21:48:35.339 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
21:48:35.351 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasktmwPRy/test-tmp
21:48:35.355 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:48:35.359 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasktmwPRy/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:48:35.602 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
21:48:35.757 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:48:35.758 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:48:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:35.762372 979 mini_oidc.cc:150] Starting JWKS server
21:48:35.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:35.766633 979 webserver.cc:480] Webserver started at http://127.0.0.1:37491/ using document root <none> and password file <none>
21:48:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:35.767217 979 mini_oidc.cc:186] Starting OIDC Discovery server
21:48:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:35.768980 979 webserver.cc:480] Webserver started at http://127.0.0.1:46739/ using document root <none> and password file <none>
21:48:35.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T21:48:35Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:48:35.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T21:48:35Z Disabled control of system clock
21:48:35.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:35.833966 979 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:48:35.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:48:35.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-0/wal
21:48:35.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-0/data
21:48:35.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:48:35.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:48:35.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:48:35.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:48:35.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:48:35.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:48:35.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-0/logs
21:48:35.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-0/data/info.pb
21:48:35.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:48:35.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:48:35.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:48:35.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:48:35.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:48:35.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:48:35.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:48:35.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:48:35.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:48:35.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:48:35.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:48:35.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:48:35.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.244.254:42075
21:48:35.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.244.254
21:48:35.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:48:35.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.244.212:42521
21:48:35.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:48:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:48:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:48:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:48:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.244.254:42075,127.0.244.253:39589,127.0.244.252:35627
21:48:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:48:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37491/jwks.json with env {}
21:48:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:36.139012 991 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:48:36.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:36.139664 991 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:48:36.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:36.140367 991 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:48:36.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:36.172935 991 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:48:36.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:36.173281 991 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:48:36.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:36.173494 991 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:48:36.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:36.173681 991 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:48:36.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:36.173945 991 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:48:36.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:36.174172 991 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37491/jwks.json
21:48:36.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:36.210549 991 master_runner.cc:386] Master server non-default flags:
21:48:36.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:48:36.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.244.212:42521
21:48:36.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:48:36.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:48:36.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-0/data
21:48:36.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-0/wal
21:48:36.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:48:36.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.244.254:42075,127.0.244.253:39589,127.0.244.252:35627
21:48:36.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:48:36.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:48:36.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:48:36.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.244.254:42075
21:48:36.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:48:36.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:48:36.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:48:36.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37491/jwks.json
21:48:36.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:48:36.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:48:36.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-0/data/info.pb
21:48:36.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.244.254
21:48:36.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:48:36.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:48:36.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:48:36.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:48:36.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:48:36.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:48:36.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-0/logs
21:48:36.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:48:36.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:48:36.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:48:36.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:48:36.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:48:36.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:48:36.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:48:36.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:48:36.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:48:36.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:48:36.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:36.212124 991 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:48:36.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:36.214375 991 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:48:36.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:36.226586 997 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:37.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:36.228873 998 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:37.447031 1000 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:37.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:37.450063 999 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1217 milliseconds
21:48:37.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.450161 991 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:48:37.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.451565 991 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:48:37.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.454130 991 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:48:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.455533 991 hybrid_clock.cc:648] HybridClock initialized: now 1756849717455496 us; error 54 us; skew 500 ppm
21:48:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.456414 991 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:48:37.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.463083 991 webserver.cc:480] Webserver started at http://127.0.244.254:36901/ using document root <none> and password file <none>
21:48:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.464066 991 fs_manager.cc:362] Metadata directory not provided
21:48:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.464273 991 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:48:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.464668 991 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:48:37.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.469178 991 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-0/data/instance:
21:48:37.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "756def174160417a8a4173a38dd4b02f"
21:48:37.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:37 on dist-test-slave-q7qm"
21:48:37.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.470450 991 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-0/wal/instance:
21:48:37.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "756def174160417a8a4173a38dd4b02f"
21:48:37.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:37 on dist-test-slave-q7qm"
21:48:37.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.477993 991 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
21:48:37.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.483922 1007 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:48:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.485265 991 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
21:48:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.485700 991 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-0/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-0/wal
21:48:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "756def174160417a8a4173a38dd4b02f"
21:48:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:37 on dist-test-slave-q7qm"
21:48:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.486140 991 fs_report.cc:389] FS layout report
21:48:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:48:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-0/wal
21:48:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-0/wal
21:48:37.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-0/data/data
21:48:37.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:48:37.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:48:37.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:48:37.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:48:37.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:48:37.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:48:37.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:48:37.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:48:37.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:48:37.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:48:37.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:48:37.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.546479 991 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:48:37.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.548100 991 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:48:37.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.548547 991 kserver.cc:163] Server-wide thread pool size limit: 3276
21:48:37.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.620258 991 rpc_server.cc:307] RPC server started. Bound to: 127.0.244.254:42075
21:48:37.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.620355 1058 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.244.254:42075 every 8 connection(s)
21:48:37.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.622931 991 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-0/data/info.pb
21:48:37.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.624441 979 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 991
21:48:37.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.625015 979 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-0/wal/instance
21:48:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.629473 1059 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:48:37.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.630513 979 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:48:37.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:48:37.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-1/wal
21:48:37.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-1/data
21:48:37.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:48:37.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:48:37.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:48:37.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:48:37.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:48:37.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:48:37.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-1/logs
21:48:37.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-1/data/info.pb
21:48:37.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:48:37.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:48:37.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:48:37.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:48:37.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:48:37.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:48:37.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:48:37.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:48:37.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:48:37.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:48:37.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:48:37.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:48:37.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.244.253:39589
21:48:37.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.244.253
21:48:37.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:48:37.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.244.212:42521
21:48:37.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:48:37.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:48:37.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:48:37.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:48:37.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.244.254:42075,127.0.244.253:39589,127.0.244.252:35627
21:48:37.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:48:37.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37491/jwks.json with env {}
21:48:37.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.643481 1059 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.244.254" port: 42075 } has no permanent_uuid. Determining permanent_uuid...
21:48:37.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.674758 1059 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } has no permanent_uuid. Determining permanent_uuid...
21:48:37.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:37.677546 1011 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.244.253:39589: connect: Connection refused (error 111)
21:48:37.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:37.682250 1059 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.244.253:39589: Network error: Client connection negotiation failed: client connection to 127.0.244.253:39589: connect: Connection refused (error 111)
21:48:37.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.731817 1059 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } attempt: 1
21:48:37.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:37.736245 1059 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.244.253:39589: Network error: Client connection negotiation failed: client connection to 127.0.244.253:39589: connect: Connection refused (error 111)
21:48:37.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.804661 1059 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } attempt: 2
21:48:37.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:37.809180 1059 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.244.253:39589: Network error: Client connection negotiation failed: client connection to 127.0.244.253:39589: connect: Connection refused (error 111)
21:48:37.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:37.900573 1059 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } attempt: 3
21:48:37.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:37.904718 1059 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.244.253:39589: Network error: Client connection negotiation failed: client connection to 127.0.244.253:39589: connect: Connection refused (error 111)
21:48:37.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:37.951120 1061 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:48:37.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:37.951771 1061 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:48:37.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:37.952427 1061 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:48:37.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:37.984474 1061 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:48:37.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:37.984771 1061 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:48:37.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:37.984966 1061 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:48:37.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:37.985147 1061 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:48:37.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:37.985335 1061 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:48:37.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:37.985492 1061 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37491/jwks.json
21:48:38.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:38.022326 1061 master_runner.cc:386] Master server non-default flags:
21:48:38.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:48:38.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.244.212:42521
21:48:38.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:48:38.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:48:38.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-1/data
21:48:38.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-1/wal
21:48:38.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:48:38.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.244.254:42075,127.0.244.253:39589,127.0.244.252:35627
21:48:38.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:48:38.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:48:38.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:48:38.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.244.253:39589
21:48:38.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:48:38.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:48:38.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:48:38.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37491/jwks.json
21:48:38.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:48:38.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:48:38.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-1/data/info.pb
21:48:38.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.244.253
21:48:38.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:48:38.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:48:38.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:48:38.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:48:38.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:48:38.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:48:38.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-1/logs
21:48:38.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:48:38.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:48:38.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:48:38.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:48:38.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:48:38.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:48:38.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:48:38.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:48:38.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:48:38.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:48:38.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:38.023751 1061 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:48:38.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:38.025843 1061 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:48:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:38.038348 1070 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:38.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:38.048219 1059 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } attempt: 4
21:48:38.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:38.056375 1059 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.244.253:39589: Network error: Client connection negotiation failed: client connection to 127.0.244.253:39589: connect: Connection refused (error 111)
21:48:38.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:38.355996 1059 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } attempt: 5
21:48:38.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:38.363778 1059 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.244.253:39589: Network error: Client connection negotiation failed: client connection to 127.0.244.253:39589: connect: Connection refused (error 111)
21:48:38.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:38.911343 1059 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } attempt: 6
21:48:38.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:38.947607 1059 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.244.253:39589: Network error: Client connection negotiation failed: client connection to 127.0.244.253:39589: connect: Connection refused (error 111)
21:48:39.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:38.038707 1071 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:39.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:39.423637 1073 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:39.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:39.426290 1061 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.388s user 0.500s sys 0.881s
21:48:39.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:39.426692 1061 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.389s user 0.500s sys 0.881s
21:48:39.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:39.427233 1072 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1382 milliseconds
21:48:39.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.427171 1061 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:48:39.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.429293 1061 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:48:39.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.434072 1061 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:48:39.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.436059 1061 hybrid_clock.cc:648] HybridClock initialized: now 1756849719435993 us; error 59 us; skew 500 ppm
21:48:39.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.437376 1061 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:48:39.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.446817 1061 webserver.cc:480] Webserver started at http://127.0.244.253:46877/ using document root <none> and password file <none>
21:48:39.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.448304 1061 fs_manager.cc:362] Metadata directory not provided
21:48:39.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.448657 1061 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:48:39.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.449241 1061 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:48:39.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.456861 1061 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-1/data/instance:
21:48:39.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "39b164a8f9894e529743dfc3ddd706b6"
21:48:39.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:39 on dist-test-slave-q7qm"
21:48:39.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.458640 1061 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-1/wal/instance:
21:48:39.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "39b164a8f9894e529743dfc3ddd706b6"
21:48:39.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:39 on dist-test-slave-q7qm"
21:48:39.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.469543 1061 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.004s sys 0.004s
21:48:39.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.477815 1082 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:48:39.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.479291 1061 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
21:48:39.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.479823 1061 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-1/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-1/wal
21:48:39.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "39b164a8f9894e529743dfc3ddd706b6"
21:48:39.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:39 on dist-test-slave-q7qm"
21:48:39.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.480381 1061 fs_report.cc:389] FS layout report
21:48:39.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:48:39.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-1/wal
21:48:39.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-1/wal
21:48:39.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-1/data/data
21:48:39.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:48:39.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:48:39.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:48:39.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:48:39.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:48:39.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:48:39.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:48:39.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:48:39.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:48:39.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:48:39.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:48:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.566540 1061 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:48:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.568076 1061 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:48:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.568502 1061 kserver.cc:163] Server-wide thread pool size limit: 3276
21:48:39.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.643471 1061 rpc_server.cc:307] RPC server started. Bound to: 127.0.244.253:39589
21:48:39.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.643563 1133 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.244.253:39589 every 8 connection(s)
21:48:39.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.646083 1061 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-1/data/info.pb
21:48:39.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.651208 1134 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:48:39.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.652714 979 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 1061
21:48:39.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.653065 979 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-1/wal/instance
21:48:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.655069 979 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:48:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:48:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-2/wal
21:48:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-2/data
21:48:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:48:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:48:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:48:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:48:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:48:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:48:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-2/logs
21:48:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-2/data/info.pb
21:48:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:48:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:48:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:48:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:48:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:48:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:48:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:48:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:48:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:48:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:48:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:48:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:48:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.244.252:35627
21:48:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.244.252
21:48:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:48:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.244.212:42521
21:48:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:48:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:48:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:48:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:48:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.244.254:42075,127.0.244.253:39589,127.0.244.252:35627
21:48:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:48:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37491/jwks.json with env {}
21:48:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.667121 1134 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.244.254" port: 42075 } has no permanent_uuid. Determining permanent_uuid...
21:48:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.694383 1134 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } has no permanent_uuid. Determining permanent_uuid...
21:48:39.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.709678 1134 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } has no permanent_uuid. Determining permanent_uuid...
21:48:39.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:39.712419 1084 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.244.252:35627: connect: Connection refused (error 111)
21:48:39.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:39.717090 1134 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.244.252:35627: Network error: Client connection negotiation failed: client connection to 127.0.244.252:35627: connect: Connection refused (error 111)
21:48:39.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.766665 1134 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } attempt: 1
21:48:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:39.771339 1134 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.244.252:35627: Network error: Client connection negotiation failed: client connection to 127.0.244.252:35627: connect: Connection refused (error 111)
21:48:39.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.839797 1134 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } attempt: 2
21:48:39.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:39.843951 1134 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.244.252:35627: Network error: Client connection negotiation failed: client connection to 127.0.244.252:35627: connect: Connection refused (error 111)
21:48:39.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:39.935474 1134 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } attempt: 3
21:48:39.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:39.940328 1134 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.244.252:35627: Network error: Client connection negotiation failed: client connection to 127.0.244.252:35627: connect: Connection refused (error 111)
21:48:39.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:39.974527 1136 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:48:39.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:39.975147 1136 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:48:39.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:39.975829 1136 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:48:40.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:40.008217 1059 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } attempt: 7
21:48:40.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:40.013499 1136 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:48:40.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:40.013864 1136 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:48:40.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:40.014200 1136 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:48:40.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:40.014568 1136 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:48:40.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:40.014882 1136 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:48:40.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:40.015123 1136 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37491/jwks.json
21:48:40.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:40.019932 1059 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } has no permanent_uuid. Determining permanent_uuid...
21:48:40.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:40.025324 1059 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.244.252:35627: Network error: Client connection negotiation failed: client connection to 127.0.244.252:35627: connect: Connection refused (error 111)
21:48:40.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:40.054775 1136 master_runner.cc:386] Master server non-default flags:
21:48:40.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:48:40.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.244.212:42521
21:48:40.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:48:40.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:48:40.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-2/data
21:48:40.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-2/wal
21:48:40.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:48:40.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.244.254:42075,127.0.244.253:39589,127.0.244.252:35627
21:48:40.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:48:40.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:48:40.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:48:40.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.244.252:35627
21:48:40.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:48:40.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:48:40.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:48:40.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37491/jwks.json
21:48:40.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:48:40.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:48:40.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-2/data/info.pb
21:48:40.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.244.252
21:48:40.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:48:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:48:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:48:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:48:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:48:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:48:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-2/logs
21:48:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:48:40.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:48:40.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:48:40.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:48:40.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:48:40.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:48:40.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:48:40.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:48:40.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:48:40.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:48:40.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:40.056527 1136 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:48:40.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:40.058779 1136 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:48:40.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:40.072098 1148 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:40.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:40.083714 1059 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } attempt: 1
21:48:40.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:40.083825 1134 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } attempt: 4
21:48:40.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:40.095845 1059 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.244.252:35627: Network error: Client connection negotiation failed: client connection to 127.0.244.252:35627: connect: Connection refused (error 111)
21:48:40.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:40.097685 1134 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.244.252:35627: Network error: Client connection negotiation failed: client connection to 127.0.244.252:35627: connect: Connection refused (error 111)
21:48:40.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:40.177537 1059 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } attempt: 2
21:48:40.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:40.194105 1059 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.244.252:35627: Network error: Client connection negotiation failed: client connection to 127.0.244.252:35627: connect: Connection refused (error 111)
21:48:40.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:40.279744 1059 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } attempt: 3
21:48:40.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:40.288683 1059 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.244.252:35627: Network error: Client connection negotiation failed: client connection to 127.0.244.252:35627: connect: Connection refused (error 111)
21:48:40.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:40.397361 1134 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } attempt: 5
21:48:40.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:40.406594 1134 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.244.252:35627: Network error: Client connection negotiation failed: client connection to 127.0.244.252:35627: connect: Connection refused (error 111)
21:48:40.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:40.429394 1059 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } attempt: 4
21:48:40.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:40.439749 1059 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.244.252:35627: Network error: Client connection negotiation failed: client connection to 127.0.244.252:35627: connect: Connection refused (error 111)
21:48:40.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:40.723433 1059 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } attempt: 5
21:48:40.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:40.732954 1059 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.244.252:35627: Network error: Client connection negotiation failed: client connection to 127.0.244.252:35627: connect: Connection refused (error 111)
21:48:40.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:40.954216 1134 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } attempt: 6
21:48:40.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:40.981082 1134 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.244.252:35627: Network error: Client connection negotiation failed: client connection to 127.0.244.252:35627: connect: Connection refused (error 111)
21:48:41.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.285589 1059 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } attempt: 6
21:48:41.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:41.317073 1059 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.0.244.252:35627: Network error: Client connection negotiation failed: client connection to 127.0.244.252:35627: connect: Connection refused (error 111)
21:48:41.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:40.073210 1149 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:41.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:41.478317 1147 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ec0 after lost signal to thread 1136
21:48:41.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:41.673946 1147 kernel_stack_watchdog.cc:198] Thread 1136 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 402ms:
21:48:41.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:48:41.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:48:41.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:48:41.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:48:41.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:41.679656 1136 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.607s user 0.573s sys 1.001s
21:48:41.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:41.679899 1150 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1607 milliseconds
21:48:41.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:41.680125 1136 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.608s user 0.573s sys 1.001s
21:48:41.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:41.681588 1151 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:41.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.681658 1136 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:48:41.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.682945 1136 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:48:41.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.685550 1136 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:48:41.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.686923 1136 hybrid_clock.cc:648] HybridClock initialized: now 1756849721686838 us; error 97 us; skew 500 ppm
21:48:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.687844 1136 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:48:41.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.694171 1136 webserver.cc:480] Webserver started at http://127.0.244.252:42723/ using document root <none> and password file <none>
21:48:41.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.695195 1136 fs_manager.cc:362] Metadata directory not provided
21:48:41.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.695452 1136 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:48:41.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.695971 1136 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:48:41.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.700515 1136 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-2/data/instance:
21:48:41.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e133684f58ca40afabb8a1014eb61fa2"
21:48:41.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:41 on dist-test-slave-q7qm"
21:48:41.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.701776 1136 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-2/wal/instance:
21:48:41.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e133684f58ca40afabb8a1014eb61fa2"
21:48:41.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:41 on dist-test-slave-q7qm"
21:48:41.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.709193 1136 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.003s
21:48:41.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.714973 1162 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:48:41.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.716156 1136 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
21:48:41.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.716518 1136 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-2/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-2/wal
21:48:41.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e133684f58ca40afabb8a1014eb61fa2"
21:48:41.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:41 on dist-test-slave-q7qm"
21:48:41.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.716908 1136 fs_report.cc:389] FS layout report
21:48:41.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:48:41.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-2/wal
21:48:41.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-2/wal
21:48:41.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-2/data/data
21:48:41.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:48:41.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:48:41.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:48:41.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:48:41.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:48:41.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:48:41.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:48:41.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:48:41.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:48:41.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:48:41.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:48:41.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.774070 1136 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:48:41.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.775717 1136 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:48:41.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.776226 1136 kserver.cc:163] Server-wide thread pool size limit: 3276
21:48:41.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.845906 1136 rpc_server.cc:307] RPC server started. Bound to: 127.0.244.252:35627
21:48:41.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.846052 1213 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.244.252:35627 every 8 connection(s)
21:48:41.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.849220 1136 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-2/data/info.pb
21:48:41.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.849797 979 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 1136
21:48:41.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.850289 979 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/master-2/wal/instance
21:48:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.857286 1214 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:48:41.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.880054 1214 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.244.254" port: 42075 } has no permanent_uuid. Determining permanent_uuid...
21:48:41.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.904989 1214 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } has no permanent_uuid. Determining permanent_uuid...
21:48:41.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.916870 1214 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } has no permanent_uuid. Determining permanent_uuid...
21:48:41.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.933212 1214 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2: Bootstrap starting.
21:48:41.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.940068 1214 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2: Neither blocks nor log segments found. Creating new log.
21:48:41.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.942004 1214 log.cc:826] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2: Log is configured to *not* fsync() on all Append() calls
21:48:41.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.947265 1214 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2: No bootstrap required, opened a new log
21:48:41.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.965978 1214 raft_consensus.cc:357] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "756def174160417a8a4173a38dd4b02f" member_type: VOTER last_known_addr { host: "127.0.244.254" port: 42075 } } peers { permanent_uuid: "39b164a8f9894e529743dfc3ddd706b6" member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } } peers { permanent_uuid: "e133684f58ca40afabb8a1014eb61fa2" member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } }
21:48:41.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.966756 1214 raft_consensus.cc:383] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:48:41.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.967051 1214 raft_consensus.cc:738] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e133684f58ca40afabb8a1014eb61fa2, State: Initialized, Role: FOLLOWER
21:48:41.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.968271 1214 consensus_queue.cc:260] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "756def174160417a8a4173a38dd4b02f" member_type: VOTER last_known_addr { host: "127.0.244.254" port: 42075 } } peers { permanent_uuid: "39b164a8f9894e529743dfc3ddd706b6" member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } } peers { permanent_uuid: "e133684f58ca40afabb8a1014eb61fa2" member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } }
21:48:41.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.970752 1223 sys_catalog.cc:455] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "756def174160417a8a4173a38dd4b02f" member_type: VOTER last_known_addr { host: "127.0.244.254" port: 42075 } } peers { permanent_uuid: "39b164a8f9894e529743dfc3ddd706b6" member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } } peers { permanent_uuid: "e133684f58ca40afabb8a1014eb61fa2" member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } } }
21:48:41.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.971395 1223 sys_catalog.cc:458] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2 [sys.catalog]: This master's current role is: FOLLOWER
21:48:41.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.972155 1214 sys_catalog.cc:564] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2 [sys.catalog]: configured and running, proceeding with master startup.
21:48:41.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:41.999432 1234 catalog_manager.cc:1560] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:48:42.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:41.999836 1234 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:48:42.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:41.992419 1136 master_runner.cc:186] Error getting master registration for 127.0.244.254:42075: OK, instance_id { permanent_uuid: "756def174160417a8a4173a38dd4b02f" instance_seqno: 1756849717600407 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:48:42.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.016718 1136 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:48:42.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:42.031302 1168 tablet.cc:2378] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:48:42.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.041653 1134 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } attempt: 7
21:48:42.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.061255 1134 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6: Bootstrap starting.
21:48:42.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.067574 1134 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6: Neither blocks nor log segments found. Creating new log.
21:48:42.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.070199 1134 log.cc:826] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6: Log is configured to *not* fsync() on all Append() calls
21:48:42.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.074934 1134 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6: No bootstrap required, opened a new log
21:48:42.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.094048 1134 raft_consensus.cc:357] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "756def174160417a8a4173a38dd4b02f" member_type: VOTER last_known_addr { host: "127.0.244.254" port: 42075 } } peers { permanent_uuid: "39b164a8f9894e529743dfc3ddd706b6" member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } } peers { permanent_uuid: "e133684f58ca40afabb8a1014eb61fa2" member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } }
21:48:42.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.094796 1134 raft_consensus.cc:383] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:48:42.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.094993 1134 raft_consensus.cc:738] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 39b164a8f9894e529743dfc3ddd706b6, State: Initialized, Role: FOLLOWER
21:48:42.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.095748 1134 consensus_queue.cc:260] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "756def174160417a8a4173a38dd4b02f" member_type: VOTER last_known_addr { host: "127.0.244.254" port: 42075 } } peers { permanent_uuid: "39b164a8f9894e529743dfc3ddd706b6" member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } } peers { permanent_uuid: "e133684f58ca40afabb8a1014eb61fa2" member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } }
21:48:42.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.098390 1238 sys_catalog.cc:455] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "756def174160417a8a4173a38dd4b02f" member_type: VOTER last_known_addr { host: "127.0.244.254" port: 42075 } } peers { permanent_uuid: "39b164a8f9894e529743dfc3ddd706b6" member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } } peers { permanent_uuid: "e133684f58ca40afabb8a1014eb61fa2" member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } } }
21:48:42.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.099195 1238 sys_catalog.cc:458] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [sys.catalog]: This master's current role is: FOLLOWER
21:48:42.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.100505 1134 sys_catalog.cc:564] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [sys.catalog]: configured and running, proceeding with master startup.
21:48:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:42.130805 1249 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:48:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:42.131656 1249 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:48:42.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.125857 1061 master_runner.cc:186] Error getting master registration for 127.0.244.254:42075: OK, instance_id { permanent_uuid: "756def174160417a8a4173a38dd4b02f" instance_seqno: 1756849717600407 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:48:42.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.163946 1061 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:48:42.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.248224 1238 raft_consensus.cc:491] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:48:42.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.248816 1238 raft_consensus.cc:513] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "756def174160417a8a4173a38dd4b02f" member_type: VOTER last_known_addr { host: "127.0.244.254" port: 42075 } } peers { permanent_uuid: "39b164a8f9894e529743dfc3ddd706b6" member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } } peers { permanent_uuid: "e133684f58ca40afabb8a1014eb61fa2" member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } }
21:48:42.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.251564 1238 leader_election.cc:290] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 756def174160417a8a4173a38dd4b02f (127.0.244.254:42075), e133684f58ca40afabb8a1014eb61fa2 (127.0.244.252:35627)
21:48:42.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.253605 1189 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "39b164a8f9894e529743dfc3ddd706b6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e133684f58ca40afabb8a1014eb61fa2" is_pre_election: true
21:48:42.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.254494 1189 raft_consensus.cc:2466] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 39b164a8f9894e529743dfc3ddd706b6 in term 0.
21:48:42.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.255899 1084 leader_election.cc:304] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 39b164a8f9894e529743dfc3ddd706b6, e133684f58ca40afabb8a1014eb61fa2; no voters:
21:48:42.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.257798 1238 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:48:42.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.258159 1238 raft_consensus.cc:491] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:48:42.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.258491 1238 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [term 0 FOLLOWER]: Advancing to term 1
21:48:42.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.271859 1238 raft_consensus.cc:513] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "756def174160417a8a4173a38dd4b02f" member_type: VOTER last_known_addr { host: "127.0.244.254" port: 42075 } } peers { permanent_uuid: "39b164a8f9894e529743dfc3ddd706b6" member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } } peers { permanent_uuid: "e133684f58ca40afabb8a1014eb61fa2" member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } }
21:48:42.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.277546 1238 leader_election.cc:290] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [CANDIDATE]: Term 1 election: Requested vote from peers 756def174160417a8a4173a38dd4b02f (127.0.244.254:42075), e133684f58ca40afabb8a1014eb61fa2 (127.0.244.252:35627)
21:48:42.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.278975 1189 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "39b164a8f9894e529743dfc3ddd706b6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e133684f58ca40afabb8a1014eb61fa2"
21:48:42.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.279587 1189 raft_consensus.cc:3058] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2 [term 0 FOLLOWER]: Advancing to term 1
21:48:42.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.287355 1189 raft_consensus.cc:2466] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 39b164a8f9894e529743dfc3ddd706b6 in term 1.
21:48:42.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.288570 1084 leader_election.cc:304] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 39b164a8f9894e529743dfc3ddd706b6, e133684f58ca40afabb8a1014eb61fa2; no voters:
21:48:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.290366 1238 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [term 1 FOLLOWER]: Leader election won for term 1
21:48:42.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.293104 1238 raft_consensus.cc:695] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [term 1 LEADER]: Becoming Leader. State: Replica: 39b164a8f9894e529743dfc3ddd706b6, State: Running, Role: LEADER
21:48:42.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.295179 1238 consensus_queue.cc:237] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "756def174160417a8a4173a38dd4b02f" member_type: VOTER last_known_addr { host: "127.0.244.254" port: 42075 } } peers { permanent_uuid: "39b164a8f9894e529743dfc3ddd706b6" member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } } peers { permanent_uuid: "e133684f58ca40afabb8a1014eb61fa2" member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } }
21:48:42.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.303670 1251 sys_catalog.cc:455] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 39b164a8f9894e529743dfc3ddd706b6. Latest consensus state: current_term: 1 leader_uuid: "39b164a8f9894e529743dfc3ddd706b6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "756def174160417a8a4173a38dd4b02f" member_type: VOTER last_known_addr { host: "127.0.244.254" port: 42075 } } peers { permanent_uuid: "39b164a8f9894e529743dfc3ddd706b6" member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } } peers { permanent_uuid: "e133684f58ca40afabb8a1014eb61fa2" member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } } }
21:48:42.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.304831 1251 sys_catalog.cc:458] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [sys.catalog]: This master's current role is: LEADER
21:48:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.311955 1253 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
21:48:42.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.324103 1253 catalog_manager.cc:1486] Initializing Kudu cluster ID...
21:48:42.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:42.327060 1088 tablet.cc:2378] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:48:42.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.342279 1189 raft_consensus.cc:1273] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2 [term 1 FOLLOWER]: Refusing update from remote peer 39b164a8f9894e529743dfc3ddd706b6: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:48:42.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.344377 1251 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e133684f58ca40afabb8a1014eb61fa2" member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:48:42.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.350610 1059 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } attempt: 7
21:48:42.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.390470 1254 mvcc.cc:204] Tried to move back new op lower bound from 7196056462697148416 to 7196056462532362240. Current Snapshot: MvccSnapshot[applied={T|T < 7196056462697148416}]
21:48:42.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.397691 1223 sys_catalog.cc:455] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 39b164a8f9894e529743dfc3ddd706b6. Latest consensus state: current_term: 1 leader_uuid: "39b164a8f9894e529743dfc3ddd706b6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "756def174160417a8a4173a38dd4b02f" member_type: VOTER last_known_addr { host: "127.0.244.254" port: 42075 } } peers { permanent_uuid: "39b164a8f9894e529743dfc3ddd706b6" member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } } peers { permanent_uuid: "e133684f58ca40afabb8a1014eb61fa2" member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } } }
21:48:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.398428 1223 sys_catalog.cc:458] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2 [sys.catalog]: This master's current role is: FOLLOWER
21:48:42.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.414332 1251 sys_catalog.cc:455] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "39b164a8f9894e529743dfc3ddd706b6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "756def174160417a8a4173a38dd4b02f" member_type: VOTER last_known_addr { host: "127.0.244.254" port: 42075 } } peers { permanent_uuid: "39b164a8f9894e529743dfc3ddd706b6" member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } } peers { permanent_uuid: "e133684f58ca40afabb8a1014eb61fa2" member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } } }
21:48:42.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.415107 1251 sys_catalog.cc:458] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [sys.catalog]: This master's current role is: LEADER
21:48:42.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.417424 1253 catalog_manager.cc:1349] Generated new cluster ID: 36de05b8bfb948a38a9c8ddc1d62cc47
21:48:42.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.417738 1253 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
21:48:42.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.418270 1059 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 756def174160417a8a4173a38dd4b02f: Bootstrap starting.
21:48:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.432725 1223 sys_catalog.cc:455] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "39b164a8f9894e529743dfc3ddd706b6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "756def174160417a8a4173a38dd4b02f" member_type: VOTER last_known_addr { host: "127.0.244.254" port: 42075 } } peers { permanent_uuid: "39b164a8f9894e529743dfc3ddd706b6" member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } } peers { permanent_uuid: "e133684f58ca40afabb8a1014eb61fa2" member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } } }
21:48:42.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.436127 1223 sys_catalog.cc:458] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2 [sys.catalog]: This master's current role is: FOLLOWER
21:48:42.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.437646 1059 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 756def174160417a8a4173a38dd4b02f: Neither blocks nor log segments found. Creating new log.
21:48:42.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.441035 1059 log.cc:826] T 00000000000000000000000000000000 P 756def174160417a8a4173a38dd4b02f: Log is configured to *not* fsync() on all Append() calls
21:48:42.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.447846 1059 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 756def174160417a8a4173a38dd4b02f: No bootstrap required, opened a new log
21:48:42.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.460819 1253 catalog_manager.cc:1372] Generated new certificate authority record
21:48:42.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.462976 1253 catalog_manager.cc:1506] Loading token signing keys...
21:48:42.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.482585 1059 raft_consensus.cc:357] T 00000000000000000000000000000000 P 756def174160417a8a4173a38dd4b02f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "756def174160417a8a4173a38dd4b02f" member_type: VOTER last_known_addr { host: "127.0.244.254" port: 42075 } } peers { permanent_uuid: "39b164a8f9894e529743dfc3ddd706b6" member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } } peers { permanent_uuid: "e133684f58ca40afabb8a1014eb61fa2" member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } }
21:48:42.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.483505 1059 raft_consensus.cc:383] T 00000000000000000000000000000000 P 756def174160417a8a4173a38dd4b02f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:48:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.483798 1059 raft_consensus.cc:738] T 00000000000000000000000000000000 P 756def174160417a8a4173a38dd4b02f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 756def174160417a8a4173a38dd4b02f, State: Initialized, Role: FOLLOWER
21:48:42.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.484767 1059 consensus_queue.cc:260] T 00000000000000000000000000000000 P 756def174160417a8a4173a38dd4b02f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "756def174160417a8a4173a38dd4b02f" member_type: VOTER last_known_addr { host: "127.0.244.254" port: 42075 } } peers { permanent_uuid: "39b164a8f9894e529743dfc3ddd706b6" member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } } peers { permanent_uuid: "e133684f58ca40afabb8a1014eb61fa2" member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } }
21:48:42.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.493021 1262 sys_catalog.cc:455] T 00000000000000000000000000000000 P 756def174160417a8a4173a38dd4b02f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "756def174160417a8a4173a38dd4b02f" member_type: VOTER last_known_addr { host: "127.0.244.254" port: 42075 } } peers { permanent_uuid: "39b164a8f9894e529743dfc3ddd706b6" member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } } peers { permanent_uuid: "e133684f58ca40afabb8a1014eb61fa2" member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } } }
21:48:42.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.493832 1262 sys_catalog.cc:458] T 00000000000000000000000000000000 P 756def174160417a8a4173a38dd4b02f [sys.catalog]: This master's current role is: FOLLOWER
21:48:42.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.497000 1059 sys_catalog.cc:564] T 00000000000000000000000000000000 P 756def174160417a8a4173a38dd4b02f [sys.catalog]: configured and running, proceeding with master startup.
21:48:42.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.498785 1032 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 756def174160417a8a4173a38dd4b02f [term 0 FOLLOWER]: Advancing to term 1
21:48:42.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.502136 1034 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "39b164a8f9894e529743dfc3ddd706b6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "756def174160417a8a4173a38dd4b02f" is_pre_election: true
21:48:42.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.508405 1033 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "39b164a8f9894e529743dfc3ddd706b6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "756def174160417a8a4173a38dd4b02f"
21:48:42.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.516629 1032 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 756def174160417a8a4173a38dd4b02f [term 1 FOLLOWER]: Refusing update from remote peer 39b164a8f9894e529743dfc3ddd706b6: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:48:42.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.518734 1251 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "756def174160417a8a4173a38dd4b02f" member_type: VOTER last_known_addr { host: "127.0.244.254" port: 42075 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:48:42.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.549360 1253 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6: Generated new TSK 0
21:48:42.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.550566 1253 catalog_manager.cc:1516] Initializing in-progress tserver states...
21:48:42.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:42.559355 1013 tablet.cc:2378] T 00000000000000000000000000000000 P 756def174160417a8a4173a38dd4b02f: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:48:42.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.590308 979 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:48:42.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:48:42.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-0/wal
21:48:42.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-0/data
21:48:42.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:48:42.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:48:42.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:48:42.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:48:42.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:48:42.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:48:42.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-0/logs
21:48:42.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-0/data/info.pb
21:48:42.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:48:42.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:48:42.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:48:42.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:48:42.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:48:42.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:48:42.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:48:42.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:48:42.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:48:42.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:48:42.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.244.193:0
21:48:42.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.244.193
21:48:42.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.244.193
21:48:42.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:48:42.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.244.254:42075,127.0.244.253:39589,127.0.244.252:35627
21:48:42.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.244.212:42521
21:48:42.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:48:42.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:48:42.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:48:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.602828 1238 sys_catalog.cc:455] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "39b164a8f9894e529743dfc3ddd706b6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "756def174160417a8a4173a38dd4b02f" member_type: VOTER last_known_addr { host: "127.0.244.254" port: 42075 } } peers { permanent_uuid: "39b164a8f9894e529743dfc3ddd706b6" member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } } peers { permanent_uuid: "e133684f58ca40afabb8a1014eb61fa2" member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } } }
21:48:42.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.603617 1238 sys_catalog.cc:458] T 00000000000000000000000000000000 P 39b164a8f9894e529743dfc3ddd706b6 [sys.catalog]: This master's current role is: LEADER
21:48:42.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.605708 1262 sys_catalog.cc:455] T 00000000000000000000000000000000 P 756def174160417a8a4173a38dd4b02f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 39b164a8f9894e529743dfc3ddd706b6. Latest consensus state: current_term: 1 leader_uuid: "39b164a8f9894e529743dfc3ddd706b6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "756def174160417a8a4173a38dd4b02f" member_type: VOTER last_known_addr { host: "127.0.244.254" port: 42075 } } peers { permanent_uuid: "39b164a8f9894e529743dfc3ddd706b6" member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } } peers { permanent_uuid: "e133684f58ca40afabb8a1014eb61fa2" member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } } }
21:48:42.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.606432 1262 sys_catalog.cc:458] T 00000000000000000000000000000000 P 756def174160417a8a4173a38dd4b02f [sys.catalog]: This master's current role is: FOLLOWER
21:48:42.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.607443 1270 sys_catalog.cc:455] T 00000000000000000000000000000000 P 756def174160417a8a4173a38dd4b02f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "39b164a8f9894e529743dfc3ddd706b6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "756def174160417a8a4173a38dd4b02f" member_type: VOTER last_known_addr { host: "127.0.244.254" port: 42075 } } peers { permanent_uuid: "39b164a8f9894e529743dfc3ddd706b6" member_type: VOTER last_known_addr { host: "127.0.244.253" port: 39589 } } peers { permanent_uuid: "e133684f58ca40afabb8a1014eb61fa2" member_type: VOTER last_known_addr { host: "127.0.244.252" port: 35627 } } }
21:48:42.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.610061 1270 sys_catalog.cc:458] T 00000000000000000000000000000000 P 756def174160417a8a4173a38dd4b02f [sys.catalog]: This master's current role is: FOLLOWER
21:48:42.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:42.680081 1279 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
21:48:43.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:43.006224 1234 catalog_manager.cc:1261] Loaded cluster ID: 36de05b8bfb948a38a9c8ddc1d62cc47
21:48:43.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:43.006932 1234 catalog_manager.cc:1554] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2: loading cluster ID for follower catalog manager: success
21:48:43.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:43.014963 1234 catalog_manager.cc:1576] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2: acquiring CA information for follower catalog manager: success
21:48:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:43.021418 1234 catalog_manager.cc:1604] T 00000000000000000000000000000000 P e133684f58ca40afabb8a1014eb61fa2: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:48:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:43.078258 1274 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:48:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:43.078817 1274 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:48:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:43.079447 1274 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:48:43.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:43.114217 1274 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:48:43.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:43.115322 1274 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.244.193
21:48:43.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:43.152967 1274 tablet_server_runner.cc:78] Tablet server non-default flags:
21:48:43.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:48:43.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.244.212:42521
21:48:43.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:48:43.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:48:43.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-0/data
21:48:43.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-0/wal
21:48:43.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:48:43.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:48:43.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.244.193:0
21:48:43.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:48:43.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:48:43.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:48:43.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-0/data/info.pb
21:48:43.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.244.193
21:48:43.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:48:43.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.244.254:42075,127.0.244.253:39589,127.0.244.252:35627
21:48:43.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:48:43.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:48:43.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:48:43.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:48:43.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:48:43.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.244.193
21:48:43.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-0/logs
21:48:43.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:48:43.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:48:43.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:48:43.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:48:43.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:48:43.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:48:43.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:48:43.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:48:43.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:48:43.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:48:43.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:43.154758 1274 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:48:43.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:43.157111 1274 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:48:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:43.173177 1285 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:43.688913 1279 catalog_manager.cc:1261] Loaded cluster ID: 36de05b8bfb948a38a9c8ddc1d62cc47
21:48:43.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:43.689247 1279 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 756def174160417a8a4173a38dd4b02f: loading cluster ID for follower catalog manager: success
21:48:43.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:43.705164 1279 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 756def174160417a8a4173a38dd4b02f: acquiring CA information for follower catalog manager: success
21:48:43.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:43.732336 1279 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 756def174160417a8a4173a38dd4b02f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:48:45.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:44.576007 1284 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 1274
21:48:45.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:45.687814 1274 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.512s user 0.821s sys 1.281s
21:48:45.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:45.698707 1274 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.523s user 0.821s sys 1.281s
21:48:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:43.177359 1286 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:45.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:45.715899 1293 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.728075 1274 server_base.cc:1047] running on GCE node
21:48:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.729602 1274 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:48:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.738389 1274 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:48:45.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.739889 1274 hybrid_clock.cc:648] HybridClock initialized: now 1756849725739632 us; error 268 us; skew 500 ppm
21:48:45.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.741021 1274 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:48:45.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.749495 1274 webserver.cc:480] Webserver started at http://127.0.244.193:34005/ using document root <none> and password file <none>
21:48:45.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.750790 1274 fs_manager.cc:362] Metadata directory not provided
21:48:45.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.751086 1274 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:48:45.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.751683 1274 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:48:45.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.758337 1274 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-0/data/instance:
21:48:45.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e4ec975e58bf4a2a947c1754a008b06d"
21:48:45.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:45 on dist-test-slave-q7qm"
21:48:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.759930 1274 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-0/wal/instance:
21:48:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e4ec975e58bf4a2a947c1754a008b06d"
21:48:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:45 on dist-test-slave-q7qm"
21:48:45.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.770040 1274 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.004s sys 0.004s
21:48:45.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.781122 1299 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:48:45.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.783190 1274 fs_manager.cc:730] Time spent opening block manager: real 0.008s user 0.002s sys 0.003s
21:48:45.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.783685 1274 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-0/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-0/wal
21:48:45.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e4ec975e58bf4a2a947c1754a008b06d"
21:48:45.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:45 on dist-test-slave-q7qm"
21:48:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.784211 1274 fs_report.cc:389] FS layout report
21:48:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:48:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-0/wal
21:48:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-0/wal
21:48:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-0/data/data
21:48:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:48:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:48:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:48:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:48:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:48:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:48:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:48:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:48:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:48:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:48:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:48:45.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.861795 1274 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:48:45.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.863445 1274 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:48:45.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.863978 1274 kserver.cc:163] Server-wide thread pool size limit: 3276
21:48:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.867141 1274 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:48:45.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.873252 1274 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:48:45.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.873566 1274 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:48:45.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.873955 1274 ts_tablet_manager.cc:610] Registered 0 tablets
21:48:45.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:45.874169 1274 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:48:46.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:46.057005 1274 rpc_server.cc:307] RPC server started. Bound to: 127.0.244.193:34981
21:48:46.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:46.057176 1411 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.244.193:34981 every 8 connection(s)
21:48:46.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:46.059729 1274 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-0/data/info.pb
21:48:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:46.071211 979 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 1274
21:48:46.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:46.072055 979 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-0/wal/instance
21:48:46.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:46.090201 979 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:48:46.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:48:46.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-1/wal
21:48:46.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-1/data
21:48:46.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:48:46.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:48:46.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:48:46.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:48:46.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:48:46.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:48:46.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-1/logs
21:48:46.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-1/data/info.pb
21:48:46.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:48:46.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:48:46.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:48:46.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:48:46.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:48:46.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:48:46.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:48:46.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:48:46.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:48:46.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:48:46.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.244.194:0
21:48:46.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.244.194
21:48:46.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.244.194
21:48:46.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:48:46.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.244.254:42075,127.0.244.253:39589,127.0.244.252:35627
21:48:46.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.244.212:42521
21:48:46.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:48:46.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:48:46.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:48:46.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:46.117753 1412 heartbeater.cc:344] Connected to a master server at 127.0.244.252:35627
21:48:46.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:46.119732 1412 heartbeater.cc:461] Registering TS with master...
21:48:46.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:46.121315 1412 heartbeater.cc:507] Master 127.0.244.252:35627 requested a full tablet report, sending...
21:48:46.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:46.125747 1179 ts_manager.cc:194] Registered new tserver with Master: e4ec975e58bf4a2a947c1754a008b06d (127.0.244.193:34981)
21:48:46.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:46.131105 1413 heartbeater.cc:344] Connected to a master server at 127.0.244.253:39589
21:48:46.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:46.131606 1413 heartbeater.cc:461] Registering TS with master...
21:48:46.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:46.132726 1413 heartbeater.cc:507] Master 127.0.244.253:39589 requested a full tablet report, sending...
21:48:46.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:46.134557 1414 heartbeater.cc:344] Connected to a master server at 127.0.244.254:42075
21:48:46.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:46.134975 1414 heartbeater.cc:461] Registering TS with master...
21:48:46.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:46.135855 1099 ts_manager.cc:194] Registered new tserver with Master: e4ec975e58bf4a2a947c1754a008b06d (127.0.244.193:34981)
21:48:46.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:46.136092 1414 heartbeater.cc:507] Master 127.0.244.254:42075 requested a full tablet report, sending...
21:48:46.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:46.139814 1024 ts_manager.cc:194] Registered new tserver with Master: e4ec975e58bf4a2a947c1754a008b06d (127.0.244.193:34981)
21:48:46.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:46.141294 1099 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.244.193:51693
21:48:46.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:46.547919 1422 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:48:46.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:46.548539 1422 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:48:46.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:46.548987 1422 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:48:46.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:46.584031 1422 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:48:46.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:46.584883 1422 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.244.194
21:48:46.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:46.624948 1422 tablet_server_runner.cc:78] Tablet server non-default flags:
21:48:46.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:48:46.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.244.212:42521
21:48:46.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:48:46.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:48:46.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-1/data
21:48:46.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-1/wal
21:48:46.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:48:46.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:48:46.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.244.194:0
21:48:46.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:48:46.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:48:46.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:48:46.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-1/data/info.pb
21:48:46.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.244.194
21:48:46.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:48:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.244.254:42075,127.0.244.253:39589,127.0.244.252:35627
21:48:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:48:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:48:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:48:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:48:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:48:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.244.194
21:48:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-1/logs
21:48:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:48:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:48:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:48:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:48:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:48:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:48:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:48:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:48:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:48:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:48:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:46.627214 1422 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:48:46.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:46.629876 1422 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:48:46.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:46.685820 1429 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:47.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:47.146440 1413 heartbeater.cc:499] Master 127.0.244.253:39589 was elected leader, sending a full tablet report...
21:48:48.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:48.071453 1428 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 1422
21:48:48.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:48.197674 1428 kernel_stack_watchdog.cc:198] Thread 1422 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 380ms:
21:48:48.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:48:48.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:48:48.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:48:48.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:48:48.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:48.198354 1422 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.510s user 0.559s sys 0.872s
21:48:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:48.198798 1422 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.510s user 0.559s sys 0.872s
21:48:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:46.688791 1430 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:48.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:48.200984 1432 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:48.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:48.204816 1431 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1510 milliseconds
21:48:48.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.204943 1422 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:48:48.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.206068 1422 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:48:48.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.208518 1422 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:48:48.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.209905 1422 hybrid_clock.cc:648] HybridClock initialized: now 1756849728209852 us; error 65 us; skew 500 ppm
21:48:48.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.210747 1422 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:48:48.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.217190 1422 webserver.cc:480] Webserver started at http://127.0.244.194:41479/ using document root <none> and password file <none>
21:48:48.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.218271 1422 fs_manager.cc:362] Metadata directory not provided
21:48:48.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.218565 1422 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:48:48.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.219049 1422 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:48:48.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.224500 1422 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-1/data/instance:
21:48:48.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27fc7bce53724e66ad868e81f04e8f07"
21:48:48.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:48 on dist-test-slave-q7qm"
21:48:48.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.225838 1422 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-1/wal/instance:
21:48:48.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27fc7bce53724e66ad868e81f04e8f07"
21:48:48.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:48 on dist-test-slave-q7qm"
21:48:48.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.233662 1422 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.002s sys 0.007s
21:48:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.240481 1440 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:48:48.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.241739 1422 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
21:48:48.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.242118 1422 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-1/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-1/wal
21:48:48.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27fc7bce53724e66ad868e81f04e8f07"
21:48:48.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:48 on dist-test-slave-q7qm"
21:48:48.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.242450 1422 fs_report.cc:389] FS layout report
21:48:48.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:48:48.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-1/wal
21:48:48.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-1/wal
21:48:48.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-1/data/data
21:48:48.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:48:48.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:48:48.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:48:48.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:48:48.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:48:48.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:48:48.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:48:48.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:48:48.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:48:48.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:48:48.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:48:48.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.302192 1422 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:48:48.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.303871 1422 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:48:48.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.304291 1422 kserver.cc:163] Server-wide thread pool size limit: 3276
21:48:48.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.306804 1422 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:48:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.311573 1422 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:48:48.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.311818 1422 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:48:48.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.312070 1422 ts_tablet_manager.cc:610] Registered 0 tablets
21:48:48.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.312310 1422 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:48:48.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.451838 1422 rpc_server.cc:307] RPC server started. Bound to: 127.0.244.194:43021
21:48:48.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.451958 1552 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.244.194:43021 every 8 connection(s)
21:48:48.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.454488 1422 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-1/data/info.pb
21:48:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.456990 979 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 1422
21:48:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.457466 979 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-1/wal/instance
21:48:48.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.480304 979 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:48:48.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:48:48.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-2/wal
21:48:48.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-2/data
21:48:48.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:48:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:48:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:48:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:48:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:48:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:48:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-2/logs
21:48:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-2/data/info.pb
21:48:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:48:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:48:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:48:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:48:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:48:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:48:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:48:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:48:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:48:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:48:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.244.195:0
21:48:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.244.195
21:48:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.244.195
21:48:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:48:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.244.254:42075,127.0.244.253:39589,127.0.244.252:35627
21:48:48.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.244.212:42521
21:48:48.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:48:48.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:48:48.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:48:48.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.501925 1554 heartbeater.cc:344] Connected to a master server at 127.0.244.253:39589
21:48:48.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.502478 1554 heartbeater.cc:461] Registering TS with master...
21:48:48.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.503790 1554 heartbeater.cc:507] Master 127.0.244.253:39589 requested a full tablet report, sending...
21:48:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.506453 1099 ts_manager.cc:194] Registered new tserver with Master: 27fc7bce53724e66ad868e81f04e8f07 (127.0.244.194:43021)
21:48:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.507259 1553 heartbeater.cc:344] Connected to a master server at 127.0.244.252:35627
21:48:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.507645 1553 heartbeater.cc:461] Registering TS with master...
21:48:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.508595 1553 heartbeater.cc:507] Master 127.0.244.252:35627 requested a full tablet report, sending...
21:48:48.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.508533 1099 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.244.194:33699
21:48:48.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.513491 1179 ts_manager.cc:194] Registered new tserver with Master: 27fc7bce53724e66ad868e81f04e8f07 (127.0.244.194:43021)
21:48:48.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.516088 1555 heartbeater.cc:344] Connected to a master server at 127.0.244.254:42075
21:48:48.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.516415 1555 heartbeater.cc:461] Registering TS with master...
21:48:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.516927 1555 heartbeater.cc:507] Master 127.0.244.254:42075 requested a full tablet report, sending...
21:48:48.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.519191 1024 ts_manager.cc:194] Registered new tserver with Master: 27fc7bce53724e66ad868e81f04e8f07 (127.0.244.194:43021)
21:48:48.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:48.831012 1563 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:48:48.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:48.831650 1563 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:48:48.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:48.832343 1563 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:48:48.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:48.884266 1563 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:48:48.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:48.885244 1563 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.244.195
21:48:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.920899 1563 tablet_server_runner.cc:78] Tablet server non-default flags:
21:48:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:48:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.244.212:42521
21:48:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:48:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:48:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-2/data
21:48:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-2/wal
21:48:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:48:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:48:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.244.195:0
21:48:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:48:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:48:48.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:48:48.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-2/data/info.pb
21:48:48.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.244.195
21:48:48.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:48:48.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.244.254:42075,127.0.244.253:39589,127.0.244.252:35627
21:48:48.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:48:48.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:48:48.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:48:48.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:48:48.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:48:48.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.244.195
21:48:48.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-2/logs
21:48:48.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:48:48.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:48:48.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:48:48.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:48:48.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:48:48.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:48:48.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:48:48.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:48:48.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:48:48.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:48:48.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.922751 1563 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:48:48.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:48.924944 1563 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:48:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:48.937839 1569 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:49.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:49.512575 1554 heartbeater.cc:499] Master 127.0.244.253:39589 was elected leader, sending a full tablet report...
21:48:50.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:48.938455 1570 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:50.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:50.300540 1563 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.362s user 0.463s sys 0.885s
21:48:50.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:50.301246 1563 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.363s user 0.464s sys 0.886s
21:48:50.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:50.308890 1572 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:50.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.308970 1563 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:48:50.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:50.308892 1571 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1368 milliseconds
21:48:50.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.311188 1563 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:48:50.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.314049 1563 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:48:50.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.315738 1563 hybrid_clock.cc:648] HybridClock initialized: now 1756849730315678 us; error 59 us; skew 500 ppm
21:48:50.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.317004 1563 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:48:50.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.326514 1563 webserver.cc:480] Webserver started at http://127.0.244.195:38649/ using document root <none> and password file <none>
21:48:50.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.327912 1563 fs_manager.cc:362] Metadata directory not provided
21:48:50.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.328223 1563 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:48:50.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.328825 1563 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:48:50.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.336352 1563 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-2/data/instance:
21:48:50.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "323d900862e14d50b386505a08b6600f"
21:48:50.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:50 on dist-test-slave-q7qm"
21:48:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.338270 1563 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-2/wal/instance:
21:48:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "323d900862e14d50b386505a08b6600f"
21:48:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:50 on dist-test-slave-q7qm"
21:48:50.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.348699 1563 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.008s sys 0.004s
21:48:50.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.357316 1580 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:48:50.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.358791 1563 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.001s
21:48:50.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.359261 1563 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-2/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-2/wal
21:48:50.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "323d900862e14d50b386505a08b6600f"
21:48:50.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:50 on dist-test-slave-q7qm"
21:48:50.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.359797 1563 fs_report.cc:389] FS layout report
21:48:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:48:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-2/wal
21:48:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-2/wal
21:48:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-2/data/data
21:48:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:48:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:48:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:48:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:48:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:48:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:48:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:48:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:48:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:48:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:48:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:48:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.452306 1563 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:48:50.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.453949 1563 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:48:50.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.454480 1563 kserver.cc:163] Server-wide thread pool size limit: 3276
21:48:50.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.457350 1563 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:48:50.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.461910 1563 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:48:50.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.462144 1563 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:48:50.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.462435 1563 ts_tablet_manager.cc:610] Registered 0 tablets
21:48:50.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.462626 1563 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:48:50.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.603905 1563 rpc_server.cc:307] RPC server started. Bound to: 127.0.244.195:33713
21:48:50.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.604018 1692 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.244.195:33713 every 8 connection(s)
21:48:50.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.606604 1563 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-2/data/info.pb
21:48:50.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.608964 979 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 1563
21:48:50.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.609556 979 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster709683382660122012/ts-2/wal/instance
21:48:50.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.657912 1693 heartbeater.cc:344] Connected to a master server at 127.0.244.252:35627
21:48:50.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.658707 1693 heartbeater.cc:461] Registering TS with master...
21:48:50.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.660674 1693 heartbeater.cc:507] Master 127.0.244.252:35627 requested a full tablet report, sending...
21:48:50.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.665997 1179 ts_manager.cc:194] Registered new tserver with Master: 323d900862e14d50b386505a08b6600f (127.0.244.195:33713)
21:48:50.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.702718 1694 heartbeater.cc:344] Connected to a master server at 127.0.244.253:39589
21:48:50.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.703822 1694 heartbeater.cc:461] Registering TS with master...
21:48:50.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.704753 1694 heartbeater.cc:507] Master 127.0.244.253:39589 requested a full tablet report, sending...
21:48:50.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.707121 1099 ts_manager.cc:194] Registered new tserver with Master: 323d900862e14d50b386505a08b6600f (127.0.244.195:33713)
21:48:50.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.708973 1099 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.244.195:54731
21:48:50.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.710059 1695 heartbeater.cc:344] Connected to a master server at 127.0.244.254:42075
21:48:50.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.710413 1695 heartbeater.cc:461] Registering TS with master...
21:48:50.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.711481 1695 heartbeater.cc:507] Master 127.0.244.254:42075 requested a full tablet report, sending...
21:48:50.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.714305 1024 ts_manager.cc:194] Registered new tserver with Master: 323d900862e14d50b386505a08b6600f (127.0.244.195:33713)
21:48:50.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:50.725040 979 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
21:48:50.737 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:48:50.739 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
21:48:50.752 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.0.244.254"
port: 42075
}
bound_http_address {
host: "127.0.244.254"
port: 36901
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.0.244.253"
port: 39589
}
bound_http_address {
host: "127.0.244.253"
port: 46877
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.0.244.252"
port: 35627
}
bound_http_address {
host: "127.0.244.252"
port: 42723
}
}
}
21:48:50.756 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:48:50.759 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.0.244.193"
port: 34981
}
bound_http_address {
host: "127.0.244.193"
port: 34005
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.0.244.194"
port: 43021
}
bound_http_address {
host: "127.0.244.194"
port: 41479
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.0.244.195"
port: 33713
}
bound_http_address {
host: "127.0.244.195"
port: 38649
}
}
}
21:48:50.760 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
21:48:50.957 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52) [main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41) [main/:?]
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43) [main/:?]
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155) [main/:?]
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299) [main/:?]
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109) [main/:?]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) [junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
21:48:50.983 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
21:48:51.110 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasktmwPRy/test-tmp
21:48:51.111 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:48:51.111 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasktmwPRy/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:48:51.114 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
21:48:51.467 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:48:51.468 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:48:51.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:51.471179 1704 mini_oidc.cc:150] Starting JWKS server
21:48:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:51.474687 1704 webserver.cc:480] Webserver started at http://127.0.0.1:41107/ using document root <none> and password file <none>
21:48:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:51.475203 1704 mini_oidc.cc:186] Starting OIDC Discovery server
21:48:51.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:51.476943 1704 webserver.cc:480] Webserver started at http://127.0.0.1:34127/ using document root <none> and password file <none>
21:48:51.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
21:48:51.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/principal' for realm 'KRBTEST.COM',
21:48:51.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
21:48:51.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:48:51 dist-test-slave-q7qm krb5kdc[1716](info): setting up network...
21:48:51.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
21:48:51.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:48:51 dist-test-slave-q7qm krb5kdc[1716](info): set up 2 sockets
21:48:51.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:48:51 dist-test-slave-q7qm krb5kdc[1716](info): commencing operation
21:48:51.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
21:48:51.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:51.714330 1694 heartbeater.cc:499] Master 127.0.244.253:39589 was elected leader, sending a full tablet report...
21:48:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:53.706801 1704 mini_kdc.cc:121] Time spent starting KDC: real 2.187s user 0.004s sys 0.015s
21:48:53.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
21:48:53.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:48:53.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
21:48:53.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
21:48:53.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:48:53.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
21:48:53.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
21:48:53.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:48:53.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
21:48:53.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:48:53.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/test-user.keytab.
21:48:53.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/test-user.keytab.
21:48:53.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:48:53 dist-test-slave-q7qm krb5kdc[1716](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849733, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:48:53.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:48:53.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T21:48:53Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:48:53.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T21:48:53Z Disabled control of system clock
21:48:53.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.170.62@KRBTEST.COM; defaulting to no policy
21:48:53.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:48:53.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.170.62@KRBTEST.COM" created.
21:48:53.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:48:53.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.170.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:48:53.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.170.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:48:53.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.1.170.62@KRBTEST.COM; defaulting to no policy
21:48:53.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:48:53.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.1.170.62@KRBTEST.COM" created.
21:48:53.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:48:53.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.170.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:48:53.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.170.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:48:53.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:53.969655 1704 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:48:53.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:48:53.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/wal
21:48:53.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/data
21:48:53.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:48:53.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:48:53.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:48:53.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:48:53.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:48:53.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:48:53.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/logs
21:48:53.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/data/info.pb
21:48:53.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:48:53.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:48:53.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:48:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:48:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:48:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:48:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:48:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:48:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:48:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:48:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:48:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:48:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.170.62:44507
21:48:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.170.62
21:48:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:48:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab
21:48:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.170.62
21:48:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:48:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:48:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:48:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:48:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.170.20:37829
21:48:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:48:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:48:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:48:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:48:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.170.62:44507,127.1.170.61:38649,127.1.170.60:37453
21:48:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:48:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41107/jwks.json
21:48:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:48:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:48:54.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:54.293792 1733 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:48:54.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:54.294489 1733 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:48:54.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:54.295248 1733 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:48:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:54.329916 1733 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:48:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:54.330226 1733 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:48:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:54.330639 1733 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:48:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:54.330960 1733 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:48:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:54.331261 1733 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:48:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:54.331566 1733 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:48:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:54.331779 1733 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41107/jwks.json
21:48:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:54.371137 1733 master_runner.cc:386] Master server non-default flags:
21:48:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:48:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.170.20:37829
21:48:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:48:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:48:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/data
21:48:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/wal
21:48:54.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:48:54.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:48:54.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.170.62:44507,127.1.170.61:38649,127.1.170.60:37453
21:48:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab
21:48:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.170.62
21:48:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:48:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:48:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:48:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.170.62:44507
21:48:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:48:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:48:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:48:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41107/jwks.json
21:48:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:48:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:48:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:48:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:48:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:48:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/data/info.pb
21:48:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.170.62
21:48:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:48:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:48:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:48:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:48:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:48:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:48:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/logs
21:48:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:48:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:48:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:48:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:48:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:48:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:48:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:48:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:48:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:48:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:48:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:54.372871 1733 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:48:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:54.374384 1733 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:48:54.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:54.381996 1549 debug-util.cc:398] Leaking SignalData structure 0x7b08000b36c0 after lost signal to thread 1424
21:48:54.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:54.383318 1549 debug-util.cc:398] Leaking SignalData structure 0x7b08000acc60 after lost signal to thread 1552
21:48:54.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:54.386811 1739 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:55.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:54.388947 1740 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:55.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:54.390046 1742 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:55.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:55.738332 1741 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
21:48:55.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.738368 1733 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:48:55.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.744657 1733 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:48:55.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.748914 1733 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:48:55.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.750746 1733 hybrid_clock.cc:648] HybridClock initialized: now 1756849735750665 us; error 93 us; skew 500 ppm
21:48:55.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.751859 1733 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:48:55.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:48:55 dist-test-slave-q7qm krb5kdc[1716](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849735, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:48:55.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.758080 1733 init.cc:377] Logged in from keytab as kudu/127.1.170.62@KRBTEST.COM (short username kudu)
21:48:55.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.764249 1733 webserver.cc:480] Webserver started at http://127.1.170.62:39525/ using document root <none> and password file <none>
21:48:55.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.765465 1733 fs_manager.cc:362] Metadata directory not provided
21:48:55.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.765761 1733 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:48:55.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.766182 1733 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:48:55.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.771087 1733 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/data/instance:
21:48:55.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5196da08cb5e4ed9af16b991714eedb8"
21:48:55.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:55 on dist-test-slave-q7qm"
21:48:55.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.772439 1733 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/wal/instance:
21:48:55.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5196da08cb5e4ed9af16b991714eedb8"
21:48:55.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:55 on dist-test-slave-q7qm"
21:48:55.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.780853 1733 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.004s sys 0.004s
21:48:55.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.787940 1751 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:48:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.789119 1733 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
21:48:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.789461 1733 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/wal
21:48:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5196da08cb5e4ed9af16b991714eedb8"
21:48:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:55 on dist-test-slave-q7qm"
21:48:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.789789 1733 fs_report.cc:389] FS layout report
21:48:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:48:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/wal
21:48:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/wal
21:48:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/data/data
21:48:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:48:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:48:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:48:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:48:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:48:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:48:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:48:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:48:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:48:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:48:55.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:48:55.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.870014 1733 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:48:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.878252 1733 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:48:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.878819 1733 kserver.cc:163] Server-wide thread pool size limit: 3276
21:48:55.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.988917 1733 rpc_server.cc:307] RPC server started. Bound to: 127.1.170.62:44507
21:48:55.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.989063 1802 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.170.62:44507 every 8 connection(s)
21:48:55.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.992122 1733 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/data/info.pb
21:48:56.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:55.999774 1803 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:48:56.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:56.002954 1704 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 1733
21:48:56.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:56.003532 1704 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/wal/instance
21:48:56.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.170.61@KRBTEST.COM; defaulting to no policy
21:48:56.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:56.019320 1803 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } has no permanent_uuid. Determining permanent_uuid...
21:48:56.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:48:56.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.170.61@KRBTEST.COM" created.
21:48:56.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:48:56.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.170.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:48:56.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.170.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:48:56.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:48:56 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849735, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.62@KRBTEST.COM for kudu/127.1.170.62@KRBTEST.COM
21:48:56.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.1.170.61@KRBTEST.COM; defaulting to no policy
21:48:56.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:56.085599 1803 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } has no permanent_uuid. Determining permanent_uuid...
21:48:56.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:48:56.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.1.170.61@KRBTEST.COM" created.
21:48:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:56.088831 1755 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.170.61:38649: connect: Connection refused (error 111)
21:48:56.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:56.094425 1803 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.170.61:38649: Network error: Client connection negotiation failed: client connection to 127.1.170.61:38649: connect: Connection refused (error 111)
21:48:56.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:48:56.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.170.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:48:56.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.170.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:48:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:56.113171 1704 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:48:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:48:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/wal
21:48:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/data
21:48:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:48:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:48:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:48:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:48:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/logs
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/data/info.pb
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.170.61:38649
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.170.61
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.170.61
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.170.20:37829
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.170.62:44507,127.1.170.61:38649,127.1.170.60:37453
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41107/jwks.json
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:48:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:48:56.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:56.144024 1803 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } attempt: 1
21:48:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:56.149623 1803 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.170.61:38649: Network error: Client connection negotiation failed: client connection to 127.1.170.61:38649: connect: Connection refused (error 111)
21:48:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:56.218098 1803 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } attempt: 2
21:48:56.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:56.223779 1803 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.170.61:38649: Network error: Client connection negotiation failed: client connection to 127.1.170.61:38649: connect: Connection refused (error 111)
21:48:56.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:56.315285 1803 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } attempt: 3
21:48:56.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:56.320329 1803 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.170.61:38649: Network error: Client connection negotiation failed: client connection to 127.1.170.61:38649: connect: Connection refused (error 111)
21:48:56.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:56.452054 1812 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:48:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:56.452646 1812 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:48:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:56.453356 1812 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:48:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:56.463820 1803 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } attempt: 4
21:48:56.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:56.467984 1803 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.170.61:38649: Network error: Client connection negotiation failed: client connection to 127.1.170.61:38649: connect: Connection refused (error 111)
21:48:56.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:56.488914 1812 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:48:56.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:56.489167 1812 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:48:56.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:56.489449 1812 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:48:56.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:56.489658 1812 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:48:56.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:56.489852 1812 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:48:56.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:56.490059 1812 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:48:56.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:56.490291 1812 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41107/jwks.json
21:48:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:56.530445 1812 master_runner.cc:386] Master server non-default flags:
21:48:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:48:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.170.20:37829
21:48:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:48:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:48:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/data
21:48:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/wal
21:48:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:48:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:48:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.170.62:44507,127.1.170.61:38649,127.1.170.60:37453
21:48:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab
21:48:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.170.61
21:48:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:48:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:48:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:48:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.170.61:38649
21:48:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:48:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:48:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:48:56.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41107/jwks.json
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/data/info.pb
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.170.61
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/logs
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:48:56.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:56.531958 1812 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:48:56.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:56.533051 1812 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:48:56.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:56.545863 1820 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:56.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:56.767450 1803 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } attempt: 5
21:48:56.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:56.772945 1803 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.170.61:38649: Network error: Client connection negotiation failed: client connection to 127.1.170.61:38649: connect: Connection refused (error 111)
21:48:57.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:57.320454 1803 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } attempt: 6
21:48:57.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:57.349062 1803 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.170.61:38649: Network error: Client connection negotiation failed: client connection to 127.1.170.61:38649: connect: Connection refused (error 111)
21:48:58.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:57.947625 1819 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 1812
21:48:58.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:58.150687 1823 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:58.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:58.146414 1812 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.600s user 0.627s sys 0.948s
21:48:58.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:58.151343 1812 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.605s user 0.628s sys 0.948s
21:48:58.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:56.546490 1821 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:58.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.163216 1812 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:48:58.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:58.164136 1822 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1617 milliseconds
21:48:58.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.165759 1812 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:48:58.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.170037 1812 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:48:58.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.171789 1812 hybrid_clock.cc:648] HybridClock initialized: now 1756849738171700 us; error 93 us; skew 500 ppm
21:48:58.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.173158 1812 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:48:58.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:48:58 dist-test-slave-q7qm krb5kdc[1716](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849738, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:48:58.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.182413 1812 init.cc:377] Logged in from keytab as kudu/127.1.170.61@KRBTEST.COM (short username kudu)
21:48:58.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.220435 1812 webserver.cc:480] Webserver started at http://127.1.170.61:40769/ using document root <none> and password file <none>
21:48:58.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.222534 1812 fs_manager.cc:362] Metadata directory not provided
21:48:58.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.223098 1812 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:48:58.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.223860 1812 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:48:58.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.233531 1812 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/data/instance:
21:48:58.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "680c50d5ebd8490a8173e465e7c7b077"
21:48:58.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:58 on dist-test-slave-q7qm"
21:48:58.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.236243 1812 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/wal/instance:
21:48:58.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "680c50d5ebd8490a8173e465e7c7b077"
21:48:58.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:58 on dist-test-slave-q7qm"
21:48:58.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.255395 1812 fs_manager.cc:696] Time spent creating directory manager: real 0.018s user 0.012s sys 0.004s
21:48:58.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.273689 1834 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:48:58.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.275805 1812 fs_manager.cc:730] Time spent opening block manager: real 0.014s user 0.004s sys 0.007s
21:48:58.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.276612 1812 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/wal
21:48:58.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "680c50d5ebd8490a8173e465e7c7b077"
21:48:58.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:58 on dist-test-slave-q7qm"
21:48:58.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.277495 1812 fs_report.cc:389] FS layout report
21:48:58.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:48:58.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/wal
21:48:58.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/wal
21:48:58.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/data/data
21:48:58.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:48:58.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:48:58.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:48:58.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:48:58.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:48:58.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:48:58.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:48:58.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:48:58.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:48:58.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:48:58.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:48:58.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.345371 1812 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:48:58.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.357275 1812 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:48:58.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.357733 1812 kserver.cc:163] Server-wide thread pool size limit: 3276
21:48:58.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.409646 1803 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } attempt: 7
21:48:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:58.417814 1803 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.170.61:38649: Network error: Client connection negotiation failed: client connection to 127.1.170.61:38649: connect: Connection refused (error 111)
21:48:58.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.448249 1812 rpc_server.cc:307] RPC server started. Bound to: 127.1.170.61:38649
21:48:58.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.448331 1887 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.170.61:38649 every 8 connection(s)
21:48:58.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.450883 1812 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/data/info.pb
21:48:58.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.455600 1704 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 1812
21:48:58.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.456086 1704 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/wal/instance
21:48:58.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.457260 1888 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:48:58.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.170.60@KRBTEST.COM; defaulting to no policy
21:48:58.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.472102 1888 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } has no permanent_uuid. Determining permanent_uuid...
21:48:58.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:48:58.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.170.60@KRBTEST.COM" created.
21:48:58.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:48:58 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849738, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.61@KRBTEST.COM for kudu/127.1.170.62@KRBTEST.COM
21:48:58.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:48:58.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.170.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:48:58.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.170.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:48:58.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.513537 1888 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } has no permanent_uuid. Determining permanent_uuid...
21:48:58.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.1.170.60@KRBTEST.COM; defaulting to no policy
21:48:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:48:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.1.170.60@KRBTEST.COM" created.
21:48:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:48:58 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849738, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.61@KRBTEST.COM for kudu/127.1.170.61@KRBTEST.COM
21:48:58.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.542104 1888 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } has no permanent_uuid. Determining permanent_uuid...
21:48:58.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:58.545193 1838 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.170.60:37453: connect: Connection refused (error 111)
21:48:58.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:48:58.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.170.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:48:58.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.170.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:48:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.549208 1704 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:48:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:48:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/wal
21:48:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/data
21:48:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:48:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:48:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:48:58.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:48:58.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:48:58.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:48:58.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/logs
21:48:58.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/data/info.pb
21:48:58.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:48:58.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:48:58.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:48:58.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:48:58.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:48:58.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:48:58.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:48:58.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:48:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:48:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:48:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:48:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:48:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.170.60:37453
21:48:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.170.60
21:48:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:48:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab
21:48:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.170.60
21:48:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:48:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:48:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:48:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:48:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.170.20:37829
21:48:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:48:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:48:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:48:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:48:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.170.62:44507,127.1.170.61:38649,127.1.170.60:37453
21:48:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:48:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41107/jwks.json
21:48:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:48:58.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:48:58.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:58.549537 1888 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.170.60:37453: Network error: Client connection negotiation failed: client connection to 127.1.170.60:37453: connect: Connection refused (error 111)
21:48:58.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.599054 1888 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } attempt: 1
21:48:58.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:58.603893 1888 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.170.60:37453: Network error: Client connection negotiation failed: client connection to 127.1.170.60:37453: connect: Connection refused (error 111)
21:48:58.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.672315 1888 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } attempt: 2
21:48:58.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:58.677398 1888 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.170.60:37453: Network error: Client connection negotiation failed: client connection to 127.1.170.60:37453: connect: Connection refused (error 111)
21:48:58.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.768776 1888 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } attempt: 3
21:48:58.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:58.772809 1888 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.170.60:37453: Network error: Client connection negotiation failed: client connection to 127.1.170.60:37453: connect: Connection refused (error 111)
21:48:58.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:58.902765 1898 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:48:58.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:58.903403 1898 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:48:58.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:58.904050 1898 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:48:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.916219 1888 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } attempt: 4
21:48:58.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:58.920042 1888 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.170.60:37453: Network error: Client connection negotiation failed: client connection to 127.1.170.60:37453: connect: Connection refused (error 111)
21:48:58.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:58.938805 1898 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:48:58.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:58.939054 1898 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:48:58.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:58.939477 1898 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:48:58.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:58.939713 1898 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:48:58.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:58.939908 1898 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:48:58.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:58.940093 1898 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:48:58.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:58.940263 1898 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41107/jwks.json
21:48:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.981083 1898 master_runner.cc:386] Master server non-default flags:
21:48:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:48:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.170.20:37829
21:48:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:48:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:48:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/data
21:48:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/wal
21:48:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:48:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.170.62:44507,127.1.170.61:38649,127.1.170.60:37453
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.170.60
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.170.60:37453
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41107/jwks.json
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/data/info.pb
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.170.60
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/logs
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:48:58.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.982578 1898 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:48:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:58.984076 1898 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:48:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:59.003580 1905 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:48:59.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:59.186585 1055 debug-util.cc:398] Leaking SignalData structure 0x7b0800096720 after lost signal to thread 992
21:48:59.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:59.187407 1055 debug-util.cc:398] Leaking SignalData structure 0x7b08000a4ee0 after lost signal to thread 1058
21:48:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:59.219470 1888 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } attempt: 5
21:48:59.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:59.227705 1888 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.170.60:37453: Network error: Client connection negotiation failed: client connection to 127.1.170.60:37453: connect: Connection refused (error 111)
21:48:59.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:48:59.785400 1888 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } attempt: 6
21:48:59.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:59.816087 1888 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.170.60:37453: Network error: Client connection negotiation failed: client connection to 127.1.170.60:37453: connect: Connection refused (error 111)
21:49:00.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:00.481869 1884 debug-util.cc:398] Leaking SignalData structure 0x7b0800071060 after lost signal to thread 1814
21:49:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:00.482851 1884 debug-util.cc:398] Leaking SignalData structure 0x7b08000717c0 after lost signal to thread 1887
21:49:00.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:00.508193 1803 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } attempt: 8
21:49:00.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:00 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849735, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.62@KRBTEST.COM for kudu/127.1.170.61@KRBTEST.COM
21:49:00.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:00.582353 1803 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } has no permanent_uuid. Determining permanent_uuid...
21:49:00.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:00.592852 1803 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.170.60:37453: Network error: Client connection negotiation failed: client connection to 127.1.170.60:37453: connect: Connection refused (error 111)
21:49:00.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:00.658331 1803 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } attempt: 1
21:49:00.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:00.664868 1803 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.170.60:37453: Network error: Client connection negotiation failed: client connection to 127.1.170.60:37453: connect: Connection refused (error 111)
21:49:00.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:00.718313 1803 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } attempt: 2
21:49:00.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:00.725649 1803 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.170.60:37453: Network error: Client connection negotiation failed: client connection to 127.1.170.60:37453: connect: Connection refused (error 111)
21:49:00.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:00.802214 1803 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } attempt: 3
21:49:00.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:00.810343 1803 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.170.60:37453: Network error: Client connection negotiation failed: client connection to 127.1.170.60:37453: connect: Connection refused (error 111)
21:49:00.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:00.876685 1888 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } attempt: 7
21:49:00.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:00.911010 1888 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.170.60:37453: Network error: Client connection negotiation failed: client connection to 127.1.170.60:37453: connect: Connection refused (error 111)
21:49:00.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:00.965765 1803 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } attempt: 4
21:49:00.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:00.972842 1803 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.170.60:37453: Network error: Client connection negotiation failed: client connection to 127.1.170.60:37453: connect: Connection refused (error 111)
21:49:01.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.269280 1803 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } attempt: 5
21:49:01.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:01.273083 1755 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.170.60:37453: connect: Connection refused (error 111) [suppressed 12 similar messages]
21:49:01.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:01.278225 1803 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.170.60:37453: Network error: Client connection negotiation failed: client connection to 127.1.170.60:37453: connect: Connection refused (error 111)
21:49:01.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:00.402153 1904 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 1898
21:49:01.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:01.431931 1904 kernel_stack_watchdog.cc:198] Thread 1898 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 396ms:
21:49:01.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:49:01.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:49:01.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:49:01.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:49:01.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:01.435225 1907 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2431 milliseconds
21:49:01.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:01.435530 1908 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:49:01.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:01.436414 1898 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.432s user 0.739s sys 1.351s
21:49:01.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:01.436998 1898 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.434s user 0.739s sys 1.351s
21:49:01.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.437510 1898 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:49:01.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:48:59.006191 1906 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:49:01.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.445782 1898 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:49:01.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.449348 1898 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:49:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.450955 1898 hybrid_clock.cc:648] HybridClock initialized: now 1756849741450786 us; error 181 us; skew 500 ppm
21:49:01.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.452188 1898 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:49:01.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:01 dist-test-slave-q7qm krb5kdc[1716](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849741, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:49:01.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.461589 1898 init.cc:377] Logged in from keytab as kudu/127.1.170.60@KRBTEST.COM (short username kudu)
21:49:01.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.471876 1898 webserver.cc:480] Webserver started at http://127.1.170.60:33881/ using document root <none> and password file <none>
21:49:01.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.473222 1898 fs_manager.cc:362] Metadata directory not provided
21:49:01.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.473551 1898 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:49:01.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.474141 1898 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:49:01.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.481577 1898 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/data/instance:
21:49:01.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7cf4756128994aa38141b5167adb0d27"
21:49:01.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:49:01 on dist-test-slave-q7qm"
21:49:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.483268 1898 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/wal/instance:
21:49:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7cf4756128994aa38141b5167adb0d27"
21:49:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:49:01 on dist-test-slave-q7qm"
21:49:01.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.494700 1898 fs_manager.cc:696] Time spent creating directory manager: real 0.011s user 0.005s sys 0.005s
21:49:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.503433 1925 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:49:01.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.506413 1898 fs_manager.cc:730] Time spent opening block manager: real 0.007s user 0.003s sys 0.001s
21:49:01.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.507414 1898 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/wal
21:49:01.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7cf4756128994aa38141b5167adb0d27"
21:49:01.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:49:01 on dist-test-slave-q7qm"
21:49:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.508430 1898 fs_report.cc:389] FS layout report
21:49:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:49:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/wal
21:49:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/wal
21:49:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/data/data
21:49:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:49:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:49:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:49:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:49:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:49:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:49:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:49:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:49:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:49:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:49:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:49:01.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.579246 1898 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:49:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.588595 1898 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:49:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.589222 1898 kserver.cc:163] Server-wide thread pool size limit: 3276
21:49:01.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.783309 1898 rpc_server.cc:307] RPC server started. Bound to: 127.1.170.60:37453
21:49:01.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.783356 1976 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.170.60:37453 every 8 connection(s)
21:49:01.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.788230 1898 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/data/info.pb
21:49:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.797439 1704 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 1898
21:49:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.798168 1704 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/wal/instance
21:49:01.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.799791 1803 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } attempt: 6
21:49:01.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.836325 1977 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:49:01.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:01 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849733, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.170.62@KRBTEST.COM
21:49:01.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:01 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849735, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.62@KRBTEST.COM for kudu/127.1.170.60@KRBTEST.COM
21:49:01.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.940022 1977 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } has no permanent_uuid. Determining permanent_uuid...
21:49:01.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:01 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849733, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.170.61@KRBTEST.COM
21:49:01.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.987149 1803 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8: Bootstrap starting.
21:49:01.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:01 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849741, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.60@KRBTEST.COM for kudu/127.1.170.62@KRBTEST.COM
21:49:01.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:01.997375 1803 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8: Neither blocks nor log segments found. Creating new log.
21:49:02.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.000944 1803 log.cc:826] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8: Log is configured to *not* fsync() on all Append() calls
21:49:02.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.012444 1803 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8: No bootstrap required, opened a new log
21:49:02.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:02 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849733, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.170.60@KRBTEST.COM
21:49:02.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.031064 1977 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } has no permanent_uuid. Determining permanent_uuid...
21:49:02.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.044992 1803 raft_consensus.cc:357] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } }
21:49:02.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.046352 1803 raft_consensus.cc:383] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:49:02.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.046902 1803 raft_consensus.cc:738] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5196da08cb5e4ed9af16b991714eedb8, State: Initialized, Role: FOLLOWER
21:49:02.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.048206 1803 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } }
21:49:02.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.056756 1987 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } } }
21:49:02.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.061079 1987 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [sys.catalog]: This master's current role is: FOLLOWER
21:49:02.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.060693 1803 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [sys.catalog]: configured and running, proceeding with master startup.
21:49:02.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.068351 1987 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:49:02.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.069229 1987 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } }
21:49:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.074258 1987 leader_election.cc:290] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 680c50d5ebd8490a8173e465e7c7b077 (127.1.170.61:38649), 7cf4756128994aa38141b5167adb0d27 (127.1.170.60:37453)
21:49:02.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:02 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849741, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.60@KRBTEST.COM for kudu/127.1.170.61@KRBTEST.COM
21:49:02.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.137485 1977 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } has no permanent_uuid. Determining permanent_uuid...
21:49:02.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:02.143326 1757 tablet.cc:2378] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:49:02.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:02 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849741, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.60@KRBTEST.COM for kudu/127.1.170.60@KRBTEST.COM
21:49:02.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:02.234896 1998 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:49:02.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:02.236658 1998 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:49:02.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.172600 1733 master_runner.cc:186] Error getting master registration for 127.1.170.61:38649: OK, instance_id { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" instance_seqno: 1756849738424443 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:49:02.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.263253 1733 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:49:02.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.271889 1977 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27: Bootstrap starting.
21:49:02.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.281152 1977 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27: Neither blocks nor log segments found. Creating new log.
21:49:02.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.283885 1977 log.cc:826] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27: Log is configured to *not* fsync() on all Append() calls
21:49:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.307394 1977 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27: No bootstrap required, opened a new log
21:49:02.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.423653 1977 raft_consensus.cc:357] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } }
21:49:02.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.428265 1977 raft_consensus.cc:383] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:49:02.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.429677 1977 raft_consensus.cc:738] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7cf4756128994aa38141b5167adb0d27, State: Initialized, Role: FOLLOWER
21:49:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.444268 1977 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } }
21:49:02.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.476002 1952 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5196da08cb5e4ed9af16b991714eedb8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7cf4756128994aa38141b5167adb0d27" is_pre_election: true
21:49:02.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.476950 1952 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5196da08cb5e4ed9af16b991714eedb8 in term 0.
21:49:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.478461 1755 leader_election.cc:304] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5196da08cb5e4ed9af16b991714eedb8, 7cf4756128994aa38141b5167adb0d27; no voters:
21:49:02.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.478875 1977 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [sys.catalog]: configured and running, proceeding with master startup.
21:49:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.479766 1987 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:49:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.480130 1987 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:49:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.480548 1987 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 0 FOLLOWER]: Advancing to term 1
21:49:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.474872 2001 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } } }
21:49:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.481406 2001 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [sys.catalog]: This master's current role is: FOLLOWER
21:49:02.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.489079 1987 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } }
21:49:02.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.499549 1952 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5196da08cb5e4ed9af16b991714eedb8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7cf4756128994aa38141b5167adb0d27"
21:49:02.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.500202 1952 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [term 0 FOLLOWER]: Advancing to term 1
21:49:02.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.499166 1987 leader_election.cc:290] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [CANDIDATE]: Term 1 election: Requested vote from peers 680c50d5ebd8490a8173e465e7c7b077 (127.1.170.61:38649), 7cf4756128994aa38141b5167adb0d27 (127.1.170.60:37453)
21:49:02.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.509140 1952 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5196da08cb5e4ed9af16b991714eedb8 in term 1.
21:49:02.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.510392 1755 leader_election.cc:304] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5196da08cb5e4ed9af16b991714eedb8, 7cf4756128994aa38141b5167adb0d27; no voters:
21:49:02.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.511974 1987 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 1 FOLLOWER]: Leader election won for term 1
21:49:02.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.526008 1987 raft_consensus.cc:695] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 1 LEADER]: Becoming Leader. State: Replica: 5196da08cb5e4ed9af16b991714eedb8, State: Running, Role: LEADER
21:49:02.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.529085 1987 consensus_queue.cc:237] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } }
21:49:02.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.564275 2004 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5196da08cb5e4ed9af16b991714eedb8. Latest consensus state: current_term: 1 leader_uuid: "5196da08cb5e4ed9af16b991714eedb8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } } }
21:49:02.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.565290 2004 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [sys.catalog]: This master's current role is: LEADER
21:49:02.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.568780 2007 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
21:49:02.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.599879 2007 catalog_manager.cc:1486] Initializing Kudu cluster ID...
21:49:02.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:02.608477 1931 tablet.cc:2378] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:49:02.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.654219 2015 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
21:49:02.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.760519 1952 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [term 1 FOLLOWER]: Refusing update from remote peer 5196da08cb5e4ed9af16b991714eedb8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:49:02.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.674549 1898 master_runner.cc:186] Error getting master registration for 127.1.170.61:38649: OK, instance_id { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" instance_seqno: 1756849738424443 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:49:02.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.764196 1898 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:49:02.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.767509 2004 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
21:49:02.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.871184 2001 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5196da08cb5e4ed9af16b991714eedb8. Latest consensus state: current_term: 1 leader_uuid: "5196da08cb5e4ed9af16b991714eedb8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } } }
21:49:02.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.894098 2001 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [sys.catalog]: This master's current role is: FOLLOWER
21:49:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.917243 2018 mvcc.cc:204] Tried to move back new op lower bound from 7196056546311426048 to 7196056545475928064. Current Snapshot: MvccSnapshot[applied={T|T < 7196056546311426048}]
21:49:02.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.940716 2001 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "5196da08cb5e4ed9af16b991714eedb8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } } }
21:49:02.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.943074 2001 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [sys.catalog]: This master's current role is: FOLLOWER
21:49:02.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.960784 2017 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "5196da08cb5e4ed9af16b991714eedb8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } } }
21:49:02.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.964579 2017 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [sys.catalog]: This master's current role is: LEADER
21:49:02.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.965963 2007 catalog_manager.cc:1349] Generated new cluster ID: fabec8501a814001a48e777c37f96b91
21:49:02.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:02.966256 2007 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
21:49:03.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.001462 1888 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } attempt: 8
21:49:03.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.124809 2007 catalog_manager.cc:1372] Generated new certificate authority record
21:49:03.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.127594 2007 catalog_manager.cc:1506] Loading token signing keys...
21:49:03.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:03 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849738, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.61@KRBTEST.COM for kudu/127.1.170.60@KRBTEST.COM
21:49:03.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.171185 2007 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8: Generated new TSK 0
21:49:03.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.172322 2007 catalog_manager.cc:1516] Initializing in-progress tserver states...
21:49:03.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.228714 1888 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077: Bootstrap starting.
21:49:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.238685 1888 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077: Neither blocks nor log segments found. Creating new log.
21:49:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.241426 1888 log.cc:826] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077: Log is configured to *not* fsync() on all Append() calls
21:49:03.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.254416 1888 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077: No bootstrap required, opened a new log
21:49:03.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.170.1@KRBTEST.COM; defaulting to no policy
21:49:03.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:49:03.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.170.1@KRBTEST.COM" created.
21:49:03.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.292606 1888 raft_consensus.cc:357] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } }
21:49:03.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.293779 1888 raft_consensus.cc:383] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:49:03.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.294227 1888 raft_consensus.cc:738] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 680c50d5ebd8490a8173e465e7c7b077, State: Initialized, Role: FOLLOWER
21:49:03.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.295495 1888 consensus_queue.cc:260] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } }
21:49:03.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.309242 2028 sys_catalog.cc:455] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } } }
21:49:03.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.311578 1888 sys_catalog.cc:564] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [sys.catalog]: configured and running, proceeding with master startup.
21:49:03.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.312148 1860 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5196da08cb5e4ed9af16b991714eedb8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "680c50d5ebd8490a8173e465e7c7b077"
21:49:03.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.315217 1861 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5196da08cb5e4ed9af16b991714eedb8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "680c50d5ebd8490a8173e465e7c7b077" is_pre_election: true
21:49:03.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.316622 1861 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:58128 (request call id 1) took 1244 ms. Trace:
21:49:03.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.317175 2028 sys_catalog.cc:458] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [sys.catalog]: This master's current role is: FOLLOWER
21:49:03.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.316959 1861 rpcz_store.cc:276] 0902 21:49:02.072395 (+ 0us) service_pool.cc:168] Inserting onto call queue
21:49:03.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:02.072550 (+ 155us) service_pool.cc:225] Handling call
21:49:03.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:03.316557 (+1244007us) inbound_call.cc:173] Queueing success response
21:49:03.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
21:49:03.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.314133 1859 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [term 0 FOLLOWER]: Advancing to term 1
21:49:03.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.326741 1859 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [term 1 FOLLOWER]: Refusing update from remote peer 5196da08cb5e4ed9af16b991714eedb8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:49:03.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.329205 2017 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
21:49:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:49:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.170.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:49:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.170.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:49:03.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:03.340047 1704 mini_kdc.cc:279] Time spent creating service keytab for kudu/127.1.170.1: real 0.118s user 0.001s sys 0.026s
21:49:03.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:03.370437 1840 tablet.cc:2378] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:49:03.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.1.170.1@KRBTEST.COM; defaulting to no policy
21:49:03.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:49:03.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.1.170.1@KRBTEST.COM" created.
21:49:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:49:03.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.170.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:49:03.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.170.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:49:03.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:03.450419 1704 mini_kdc.cc:279] Time spent creating service keytab for HTTP/127.1.170.1: real 0.110s user 0.000s sys 0.013s
21:49:03.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.449645 2028 sys_catalog.cc:455] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5196da08cb5e4ed9af16b991714eedb8. Latest consensus state: current_term: 1 leader_uuid: "5196da08cb5e4ed9af16b991714eedb8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } } }
21:49:03.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.451040 2028 sys_catalog.cc:458] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [sys.catalog]: This master's current role is: FOLLOWER
21:49:03.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.451768 2036 sys_catalog.cc:455] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "5196da08cb5e4ed9af16b991714eedb8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } } }
21:49:03.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.452847 1704 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:49:03.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:49:03.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-0/wal
21:49:03.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-0/data
21:49:03.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:49:03.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:49:03.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:49:03.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:49:03.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:49:03.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:49:03.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-0/logs
21:49:03.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-0/data/info.pb
21:49:03.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:49:03.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:49:03.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:49:03.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:49:03.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:49:03.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:49:03.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:49:03.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:49:03.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:49:03.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:49:03.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.170.1:0
21:49:03.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.170.1
21:49:03.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.170.1
21:49:03.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:49:03.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.170.62:44507,127.1.170.61:38649,127.1.170.60:37453
21:49:03.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab
21:49:03.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.170.1
21:49:03.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:49:03.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:49:03.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:49:03.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:49:03.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.170.20:37829
21:49:03.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:49:03.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:49:03.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:49:03.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:49:03.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.455984 2036 sys_catalog.cc:458] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [sys.catalog]: This master's current role is: FOLLOWER
21:49:03.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:03.471405 1884 messenger.cc:191] Invalid argument: socket address is ambiguous: no data on pending connections for acceptor pool at 127.1.170.61:38649
21:49:03.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.499305 2048 catalog_manager.cc:1261] Loaded cluster ID: fabec8501a814001a48e777c37f96b91
21:49:03.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.499620 2048 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077: loading cluster ID for follower catalog manager: success
21:49:03.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.505472 2048 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077: acquiring CA information for follower catalog manager: success
21:49:03.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.512024 2048 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:49:03.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.673884 2015 catalog_manager.cc:1261] Loaded cluster ID: fabec8501a814001a48e777c37f96b91
21:49:03.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.674181 2015 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27: loading cluster ID for follower catalog manager: success
21:49:03.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.688813 2015 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27: acquiring CA information for follower catalog manager: success
21:49:03.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.707105 2015 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:49:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.795702 2017 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "5196da08cb5e4ed9af16b991714eedb8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } } }
21:49:03.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:03.796794 2017 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [sys.catalog]: This master's current role is: LEADER
21:49:04.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:04.068684 1973 debug-util.cc:398] Leaking SignalData structure 0x7b0800082860 after lost signal to thread 1900
21:49:04.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:04.069362 1973 debug-util.cc:398] Leaking SignalData structure 0x7b0800082e00 after lost signal to thread 1976
21:49:04.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:04.074415 2047 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:49:04.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:04.075393 2047 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:49:04.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:04.076830 2047 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:49:04.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:04.142791 2047 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:49:04.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:04.145934 2047 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.170.1
21:49:04.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:04.220809 2047 tablet_server_runner.cc:78] Tablet server non-default flags:
21:49:04.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:49:04.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.170.20:37829
21:49:04.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:49:04.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:49:04.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-0/data
21:49:04.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-0/wal
21:49:04.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab
21:49:04.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.170.1
21:49:04.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:49:04.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:49:04.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.170.1:0
21:49:04.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:49:04.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:49:04.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:49:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:49:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:49:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:49:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-0/data/info.pb
21:49:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.170.1
21:49:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:49:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.170.62:44507,127.1.170.61:38649,127.1.170.60:37453
21:49:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:49:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:49:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:49:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:49:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:49:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.170.1
21:49:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-0/logs
21:49:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:49:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:49:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:49:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:49:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:49:04.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:49:04.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:49:04.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:49:04.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:49:04.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:49:04.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:04.224634 2047 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:49:04.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:04.226399 2047 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:49:04.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:04.258519 2058 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:49:05.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:05 dist-test-slave-q7qm krb5kdc[1716](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849745, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:49:05.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:05.764899 1747 init.cc:295] Successfully reacquired a new kerberos TGT
21:49:06.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:05.651420 2057 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 2047
21:49:06.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:06.671156 2057 kernel_stack_watchdog.cc:198] Thread 2047 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 382ms:
21:49:06.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:49:06.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:49:06.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:49:06.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:49:06.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:04.271976 2059 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:49:06.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:06.674377 2061 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:49:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:06.679232 2047 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.412s user 0.258s sys 0.568s
21:49:06.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:06.679863 2047 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.412s user 0.258s sys 0.568s
21:49:06.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:06.679965 2060 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2405 milliseconds
21:49:06.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:06.680279 2047 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:49:06.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:06.688567 2047 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:49:06.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:06.695295 2047 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:49:06.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:06.711169 2047 hybrid_clock.cc:648] HybridClock initialized: now 1756849746711105 us; error 78 us; skew 500 ppm
21:49:06.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:06.712412 2047 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:49:06.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:06 dist-test-slave-q7qm krb5kdc[1716](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849746, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:49:06.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:06.728806 2047 init.cc:377] Logged in from keytab as kudu/127.1.170.1@KRBTEST.COM (short username kudu)
21:49:06.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:06.765321 2047 webserver.cc:480] Webserver started at http://127.1.170.1:45039/ using document root <none> and password file <none>
21:49:06.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:06.766728 2047 fs_manager.cc:362] Metadata directory not provided
21:49:06.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:06.767035 2047 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:49:06.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:06.767664 2047 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:49:06.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:06.774847 2047 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-0/data/instance:
21:49:06.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8abe630437664f39816cac5f99c5c6cf"
21:49:06.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:49:06 on dist-test-slave-q7qm"
21:49:06.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:06.788654 2047 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-0/wal/instance:
21:49:06.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8abe630437664f39816cac5f99c5c6cf"
21:49:06.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:49:06 on dist-test-slave-q7qm"
21:49:06.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:06.810177 2047 fs_manager.cc:696] Time spent creating directory manager: real 0.021s user 0.005s sys 0.005s
21:49:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:06.826746 2074 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:49:06.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:06.832305 2047 fs_manager.cc:730] Time spent opening block manager: real 0.018s user 0.005s sys 0.003s
21:49:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:06.832792 2047 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-0/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-0/wal
21:49:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8abe630437664f39816cac5f99c5c6cf"
21:49:06.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:49:06 on dist-test-slave-q7qm"
21:49:06.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:06.833276 2047 fs_report.cc:389] FS layout report
21:49:06.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:49:06.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-0/wal
21:49:06.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-0/wal
21:49:06.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-0/data/data
21:49:06.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:49:06.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:49:06.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:49:06.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:49:06.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:49:06.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:49:06.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:49:06.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:49:06.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:49:06.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:49:06.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:49:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:06.982539 2047 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:49:06.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:06.998428 2047 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:49:06.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:06.999046 2047 kserver.cc:163] Server-wide thread pool size limit: 3276
21:49:07.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:07.015223 2047 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:49:07.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:07.030910 2047 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:49:07.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:07.031217 2047 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:49:07.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:07.031567 2047 ts_tablet_manager.cc:610] Registered 0 tablets
21:49:07.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:07.031790 2047 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:49:08.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:08.106251 2187 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.170.1:43877 every 8 connection(s)
21:49:08.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:08.107540 2047 rpc_server.cc:307] RPC server started. Bound to: 127.1.170.1:43877
21:49:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:08.111645 2047 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-0/data/info.pb
21:49:08.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:08.120419 1704 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 2047
21:49:08.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:08.120954 1704 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-0/wal/instance
21:49:08.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.170.2@KRBTEST.COM; defaulting to no policy
21:49:08.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:08 dist-test-slave-q7qm krb5kdc[1716](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849748, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:49:08.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:08.197968 1830 init.cc:295] Successfully reacquired a new kerberos TGT
21:49:08.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:49:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.170.2@KRBTEST.COM" created.
21:49:08.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:08 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849746, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.1@KRBTEST.COM for kudu/127.1.170.61@KRBTEST.COM
21:49:08.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:08 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849746, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.1@KRBTEST.COM for kudu/127.1.170.62@KRBTEST.COM
21:49:08.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:08 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849746, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.1@KRBTEST.COM for kudu/127.1.170.60@KRBTEST.COM
21:49:08.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:08.274788 2189 heartbeater.cc:344] Connected to a master server at 127.1.170.61:38649
21:49:08.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:08.275398 2189 heartbeater.cc:461] Registering TS with master...
21:49:08.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:08.276913 2193 heartbeater.cc:344] Connected to a master server at 127.1.170.62:44507
21:49:08.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:08.277436 2193 heartbeater.cc:461] Registering TS with master...
21:49:08.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:08.278792 2193 heartbeater.cc:507] Master 127.1.170.62:44507 requested a full tablet report, sending...
21:49:08.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:08.276909 2189 heartbeater.cc:507] Master 127.1.170.61:38649 requested a full tablet report, sending...
21:49:08.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:08.283825 1768 ts_manager.cc:194] Registered new tserver with Master: 8abe630437664f39816cac5f99c5c6cf (127.1.170.1:43877)
21:49:08.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:08.283808 1851 ts_manager.cc:194] Registered new tserver with Master: 8abe630437664f39816cac5f99c5c6cf (127.1.170.1:43877)
21:49:08.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:08.286645 1768 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.1.170.1@KRBTEST.COM'} at 127.1.170.1:47639
21:49:08.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:08.288368 2188 heartbeater.cc:344] Connected to a master server at 127.1.170.60:37453
21:49:08.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:08.288779 2188 heartbeater.cc:461] Registering TS with master...
21:49:08.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:08.289929 2188 heartbeater.cc:507] Master 127.1.170.60:37453 requested a full tablet report, sending...
21:49:08.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:08.294337 1942 ts_manager.cc:194] Registered new tserver with Master: 8abe630437664f39816cac5f99c5c6cf (127.1.170.1:43877)
21:49:08.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:49:08.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.170.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:49:08.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.170.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:49:08.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:08.300331 1704 mini_kdc.cc:279] Time spent creating service keytab for kudu/127.1.170.2: real 0.159s user 0.003s sys 0.013s
21:49:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.1.170.2@KRBTEST.COM; defaulting to no policy
21:49:08.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:49:08.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.1.170.2@KRBTEST.COM" created.
21:49:08.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:49:08.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.170.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:49:08.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.170.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:49:08.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:08.370666 1704 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:49:08.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:49:08.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-1/wal
21:49:08.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-1/data
21:49:08.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:49:08.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:49:08.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:49:08.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:49:08.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:49:08.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:49:08.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-1/logs
21:49:08.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-1/data/info.pb
21:49:08.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:49:08.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:49:08.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:49:08.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:49:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:49:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:49:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:49:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:49:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:49:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:49:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.170.2:0
21:49:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.170.2
21:49:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.170.2
21:49:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:49:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.170.62:44507,127.1.170.61:38649,127.1.170.60:37453
21:49:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab
21:49:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.170.2
21:49:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:49:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:49:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:49:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:49:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.170.20:37829
21:49:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:49:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:49:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:49:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:49:08.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:08.849411 2202 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:49:08.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:08.850317 2202 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:49:08.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:08.851483 2202 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:49:08.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:08.930235 2202 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:49:08.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:08.932788 2202 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.170.2
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:09.010385 2202 tablet_server_runner.cc:78] Tablet server non-default flags:
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.170.20:37829
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-1/data
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-1/wal
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.170.2
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.170.2:0
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-1/data/info.pb
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.170.2
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.170.62:44507,127.1.170.61:38649,127.1.170.60:37453
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:49:09.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:49:09.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:49:09.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.170.2
21:49:09.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-1/logs
21:49:09.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:49:09.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:49:09.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:49:09.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:49:09.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:49:09.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:49:09.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:49:09.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:49:09.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:49:09.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:49:09.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:09.012899 2202 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:49:09.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:09.014452 2202 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:49:09.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:09.045447 2210 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:49:09.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:09.299033 2193 heartbeater.cc:499] Master 127.1.170.62:44507 was elected leader, sending a full tablet report...
21:49:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:10.448307 2209 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 2202
21:49:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:10.571547 2202 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.528s user 0.545s sys 0.892s
21:49:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:09.047501 2211 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:49:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:10.571919 2202 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.528s user 0.545s sys 0.892s
21:49:10.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:10.581712 2217 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:49:10.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.582830 2202 server_base.cc:1047] running on GCE node
21:49:10.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.584214 2202 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:49:10.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.586701 2202 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:49:10.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.588147 2202 hybrid_clock.cc:648] HybridClock initialized: now 1756849750588089 us; error 55 us; skew 500 ppm
21:49:10.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.589188 2202 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:49:10.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:10 dist-test-slave-q7qm krb5kdc[1716](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849750, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:49:10.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.596143 2202 init.cc:377] Logged in from keytab as kudu/127.1.170.2@KRBTEST.COM (short username kudu)
21:49:10.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.601483 2202 webserver.cc:480] Webserver started at http://127.1.170.2:44291/ using document root <none> and password file <none>
21:49:10.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.602487 2202 fs_manager.cc:362] Metadata directory not provided
21:49:10.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.602705 2202 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:49:10.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.603124 2202 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:49:10.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.607602 2202 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-1/data/instance:
21:49:10.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d700b2d5bdb54547bd82dfc56a2b598f"
21:49:10.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:49:10 on dist-test-slave-q7qm"
21:49:10.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.608754 2202 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-1/wal/instance:
21:49:10.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d700b2d5bdb54547bd82dfc56a2b598f"
21:49:10.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:49:10 on dist-test-slave-q7qm"
21:49:10.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.617347 2202 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.008s sys 0.000s
21:49:10.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.622906 2224 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:49:10.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.624068 2202 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
21:49:10.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.624408 2202 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-1/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-1/wal
21:49:10.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d700b2d5bdb54547bd82dfc56a2b598f"
21:49:10.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:49:10 on dist-test-slave-q7qm"
21:49:10.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.624733 2202 fs_report.cc:389] FS layout report
21:49:10.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:49:10.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-1/wal
21:49:10.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-1/wal
21:49:10.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-1/data/data
21:49:10.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:49:10.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:49:10.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:49:10.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:49:10.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:49:10.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:49:10.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:49:10.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:49:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:49:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:49:10.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:49:10.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.678851 2202 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:49:10.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.684794 2202 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:49:10.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.685225 2202 kserver.cc:163] Server-wide thread pool size limit: 3276
21:49:10.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.688190 2202 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:49:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.692490 2202 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:49:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.692708 2202 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:49:10.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.692903 2202 ts_tablet_manager.cc:610] Registered 0 tablets
21:49:10.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.693058 2202 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:49:10.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.867283 2202 rpc_server.cc:307] RPC server started. Bound to: 127.1.170.2:33647
21:49:10.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.867542 2336 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.170.2:33647 every 8 connection(s)
21:49:10.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.870654 2202 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-1/data/info.pb
21:49:10.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.872538 1704 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 2202
21:49:10.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.873023 1704 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-1/wal/instance
21:49:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.170.3@KRBTEST.COM; defaulting to no policy
21:49:10.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:49:10.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.170.3@KRBTEST.COM" created.
21:49:10.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:10 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849750, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.2@KRBTEST.COM for kudu/127.1.170.60@KRBTEST.COM
21:49:10.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:10 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849750, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.2@KRBTEST.COM for kudu/127.1.170.62@KRBTEST.COM
21:49:10.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:10 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849750, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.2@KRBTEST.COM for kudu/127.1.170.61@KRBTEST.COM
21:49:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:49:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.170.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:49:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.170.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:49:10.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.953116 2337 heartbeater.cc:344] Connected to a master server at 127.1.170.60:37453
21:49:10.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.954226 2337 heartbeater.cc:461] Registering TS with master...
21:49:10.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.955639 2337 heartbeater.cc:507] Master 127.1.170.60:37453 requested a full tablet report, sending...
21:49:10.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.958932 1942 ts_manager.cc:194] Registered new tserver with Master: d700b2d5bdb54547bd82dfc56a2b598f (127.1.170.2:33647)
21:49:10.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.959920 2338 heartbeater.cc:344] Connected to a master server at 127.1.170.61:38649
21:49:10.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.960309 2338 heartbeater.cc:461] Registering TS with master...
21:49:10.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.961396 2338 heartbeater.cc:507] Master 127.1.170.61:38649 requested a full tablet report, sending...
21:49:10.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.964658 1851 ts_manager.cc:194] Registered new tserver with Master: d700b2d5bdb54547bd82dfc56a2b598f (127.1.170.2:33647)
21:49:10.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.965481 2341 heartbeater.cc:344] Connected to a master server at 127.1.170.62:44507
21:49:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.965842 2341 heartbeater.cc:461] Registering TS with master...
21:49:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.966640 2341 heartbeater.cc:507] Master 127.1.170.62:44507 requested a full tablet report, sending...
21:49:10.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.968660 1768 ts_manager.cc:194] Registered new tserver with Master: d700b2d5bdb54547bd82dfc56a2b598f (127.1.170.2:33647)
21:49:10.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.970013 1768 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.1.170.2@KRBTEST.COM'} at 127.1.170.2:49317
21:49:10.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.1.170.3@KRBTEST.COM; defaulting to no policy
21:49:10.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:49:10.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.1.170.3@KRBTEST.COM" created.
21:49:10.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:49:10.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.170.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:49:10.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.170.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab.
21:49:10.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:10.998509 1704 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-2/wal
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-2/data
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-2/logs
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-2/data/info.pb
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.170.3:0
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.170.3
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.170.3
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.170.62:44507,127.1.170.61:38649,127.1.170.60:37453
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.170.3
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:49:10.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.170.20:37829
21:49:11.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:49:11.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:49:11.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:49:11.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:49:11.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:11.342864 2351 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:49:11.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:11.343343 2351 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:49:11.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:11.343984 2351 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:49:11.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:11.376341 2351 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:49:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:11.377439 2351 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.170.3
21:49:11.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:11.417351 2351 tablet_server_runner.cc:78] Tablet server non-default flags:
21:49:11.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:49:11.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.170.20:37829
21:49:11.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:49:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:49:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-2/data
21:49:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-2/wal
21:49:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab
21:49:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.170.3
21:49:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:49:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:49:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.170.3:0
21:49:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:49:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:49:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:49:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:49:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:49:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:49:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-2/data/info.pb
21:49:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.170.3
21:49:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:49:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.170.62:44507,127.1.170.61:38649,127.1.170.60:37453
21:49:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:49:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:49:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:49:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:49:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:49:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.170.3
21:49:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-2/logs
21:49:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:49:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:49:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:49:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:49:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:49:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:49:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:49:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:49:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:49:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:49:11.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:11.419025 2351 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:49:11.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:11.420361 2351 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:49:11.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:11.434818 2357 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:49:11.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:11 dist-test-slave-q7qm krb5kdc[1716](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849751, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:49:11.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:11.467020 1921 init.cc:295] Successfully reacquired a new kerberos TGT
21:49:11.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:11.973662 2341 heartbeater.cc:499] Master 127.1.170.62:44507 was elected leader, sending a full tablet report...
21:49:13.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:12.838724 2356 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 2351
21:49:13.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:13.229300 2351 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.793s user 0.574s sys 1.122s
21:49:13.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:11.436492 2358 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:49:13.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:13.229707 2351 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.794s user 0.574s sys 1.122s
21:49:13.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:13.231974 2360 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:49:13.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:13.236610 2359 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1799 milliseconds
21:49:13.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.236640 2351 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:49:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.237900 2351 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:49:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.239936 2351 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:49:13.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.241305 2351 hybrid_clock.cc:648] HybridClock initialized: now 1756849753241250 us; error 67 us; skew 500 ppm
21:49:13.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.242170 2351 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:49:13.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:13 dist-test-slave-q7qm krb5kdc[1716](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849753, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:49:13.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.248449 2351 init.cc:377] Logged in from keytab as kudu/127.1.170.3@KRBTEST.COM (short username kudu)
21:49:13.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.253206 2351 webserver.cc:480] Webserver started at http://127.1.170.3:35887/ using document root <none> and password file <none>
21:49:13.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.254112 2351 fs_manager.cc:362] Metadata directory not provided
21:49:13.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.254307 2351 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:49:13.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.254700 2351 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:49:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.259500 2351 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-2/data/instance:
21:49:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "62f724de78e74385a721887545e100f3"
21:49:13.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:49:13 on dist-test-slave-q7qm"
21:49:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.260748 2351 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-2/wal/instance:
21:49:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "62f724de78e74385a721887545e100f3"
21:49:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:49:13 on dist-test-slave-q7qm"
21:49:13.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.269673 2351 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.003s
21:49:13.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.275599 2371 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:49:13.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.277035 2351 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
21:49:13.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.277377 2351 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-2/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-2/wal
21:49:13.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "62f724de78e74385a721887545e100f3"
21:49:13.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:49:13 on dist-test-slave-q7qm"
21:49:13.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.277709 2351 fs_report.cc:389] FS layout report
21:49:13.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:49:13.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-2/wal
21:49:13.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-2/wal
21:49:13.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-2/data/data
21:49:13.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:49:13.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:49:13.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:49:13.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:49:13.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:49:13.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:49:13.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:49:13.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:49:13.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:49:13.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:49:13.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:49:13.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.350036 2351 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:49:13.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.360723 2351 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:49:13.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.361184 2351 kserver.cc:163] Server-wide thread pool size limit: 3276
21:49:13.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.363806 2351 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:49:13.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.368101 2351 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:49:13.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.368317 2351 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:49:13.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.368666 2351 ts_tablet_manager.cc:610] Registered 0 tablets
21:49:13.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.368837 2351 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
21:49:13.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.603116 2351 rpc_server.cc:307] RPC server started. Bound to: 127.1.170.3:39057
21:49:13.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.603266 2483 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.170.3:39057 every 8 connection(s)
21:49:13.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.605890 2351 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-2/data/info.pb
21:49:13.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.607755 1704 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 2351
21:49:13.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.608417 1704 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/ts-2/wal/instance
21:49:13.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:13 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849753, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.3@KRBTEST.COM for kudu/127.1.170.60@KRBTEST.COM
21:49:13.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:13 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849753, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.3@KRBTEST.COM for kudu/127.1.170.62@KRBTEST.COM
21:49:13.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:13 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849753, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.3@KRBTEST.COM for kudu/127.1.170.61@KRBTEST.COM
21:49:13.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.683290 2484 heartbeater.cc:344] Connected to a master server at 127.1.170.60:37453
21:49:13.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.683851 2484 heartbeater.cc:461] Registering TS with master...
21:49:13.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.685127 2484 heartbeater.cc:507] Master 127.1.170.60:37453 requested a full tablet report, sending...
21:49:13.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.688426 1942 ts_manager.cc:194] Registered new tserver with Master: 62f724de78e74385a721887545e100f3 (127.1.170.3:39057)
21:49:13.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.691006 2488 heartbeater.cc:344] Connected to a master server at 127.1.170.62:44507
21:49:13.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.691414 2488 heartbeater.cc:461] Registering TS with master...
21:49:13.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.692415 2488 heartbeater.cc:507] Master 127.1.170.62:44507 requested a full tablet report, sending...
21:49:13.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.694396 2485 heartbeater.cc:344] Connected to a master server at 127.1.170.61:38649
21:49:13.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.694758 2485 heartbeater.cc:461] Registering TS with master...
21:49:13.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.694661 1768 ts_manager.cc:194] Registered new tserver with Master: 62f724de78e74385a721887545e100f3 (127.1.170.3:39057)
21:49:13.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.695719 2485 heartbeater.cc:507] Master 127.1.170.61:38649 requested a full tablet report, sending...
21:49:13.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.696296 1768 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.1.170.3@KRBTEST.COM'} at 127.1.170.3:46635
21:49:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.698215 1851 ts_manager.cc:194] Registered new tserver with Master: 62f724de78e74385a721887545e100f3 (127.1.170.3:39057)
21:49:13.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.715621 1704 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
21:49:13.728 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:49:13.730 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
21:49:13.741 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
21:49:13.741 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
21:49:13.745 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.170.62"
port: 44507
}
bound_http_address {
host: "127.1.170.62"
port: 39525
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.170.61"
port: 38649
}
bound_http_address {
host: "127.1.170.61"
port: 40769
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.170.60"
port: 37453
}
bound_http_address {
host: "127.1.170.60"
port: 33881
}
}
}
21:49:13.746 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:49:13.747 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.1.170.1"
port: 43877
}
bound_http_address {
host: "127.1.170.1"
port: 45039
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.1.170.2"
port: 33647
}
bound_http_address {
host: "127.1.170.2"
port: 44291
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.1.170.3"
port: 39057
}
bound_http_address {
host: "127.1.170.3"
port: 35887
}
}
}
21:49:13.747 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
21:49:13.749 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
21:49:13.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:13 dist-test-slave-q7qm krb5kdc[1716](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849753, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:49:13.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:49:13.785 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:49:13.786 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/krb5cc
21:49:13.792 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:49:13.793 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/krb5cc
21:49:13.794 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:49:13.797 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
21:49:13.799 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
21:49:13.799 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
21:49:13.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:13 dist-test-slave-q7qm krb5kdc[1716](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849753, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:49:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:49:13.830 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:49:13.830 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/krb5cc
21:49:13.832 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:49:13.832 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.170.62:44507
21:49:13.833 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
21:49:13.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.834275 1704 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 1733
21:49:13.868 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:49:13.869 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.170.60:37453
21:49:13.869 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
21:49:13.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.869588 1704 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 1898
21:49:13.904 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:49:13.904 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.170.61:38649
21:49:13.905 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
21:49:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.905422 1704 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 1812
21:49:13.934 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:49:13.935 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.170.62:44507
21:49:13.938 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
21:49:13.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:13.939949 1704 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:49:13.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:49:13.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/wal
21:49:13.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/data
21:49:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:49:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:49:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:49:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:49:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:49:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:49:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/logs
21:49:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/data/info.pb
21:49:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:49:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:49:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:49:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:49:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:49:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:49:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:49:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:49:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:49:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:49:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:49:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:49:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.170.62:44507
21:49:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.170.62
21:49:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39525
21:49:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab
21:49:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.170.62
21:49:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:49:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:49:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:49:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:49:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.170.20:37829
21:49:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:49:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:49:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:49:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:49:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.170.62:44507,127.1.170.61:38649,127.1.170.60:37453
21:49:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:49:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41107/jwks.json
21:49:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:49:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:49:13.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:13.990422 2341 heartbeater.cc:646] Failed to heartbeat to 127.1.170.62:44507 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.1.170.62:44507: connect: Connection refused (error 111)
21:49:14.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:14.274819 2498 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:49:14.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:14.275522 2498 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:49:14.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:14.276183 2498 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:49:14.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:14.312141 2498 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:49:14.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:14.312402 2498 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:49:14.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:14.312677 2498 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:49:14.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:14.312880 2498 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:49:14.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:14.313071 2498 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:49:14.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:14.313256 2498 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:49:14.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:14.313406 2498 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41107/jwks.json
21:49:14.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:14.337124 2193 heartbeater.cc:646] Failed to heartbeat to 127.1.170.62:44507 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.1.170.62:44507: connect: Connection refused (error 111)
21:49:14.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:14.358431 2498 master_runner.cc:386] Master server non-default flags:
21:49:14.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:49:14.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.170.20:37829
21:49:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:49:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:49:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/data
21:49:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/wal
21:49:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:49:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:49:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.170.62:44507,127.1.170.61:38649,127.1.170.60:37453
21:49:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab
21:49:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.170.62
21:49:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:49:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:49:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:49:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.170.62:44507
21:49:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:49:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:49:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:49:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41107/jwks.json
21:49:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:49:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:49:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:49:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:49:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:49:14.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/data/info.pb
21:49:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.170.62
21:49:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39525
21:49:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:49:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:49:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:49:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:49:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:49:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/logs
21:49:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:49:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:49:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:49:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:49:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:49:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:49:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:49:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:49:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:49:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:49:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:14.359969 2498 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:49:14.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:14.361091 2498 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:49:14.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:14.373406 2511 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:49:14.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:14.700093 2488 heartbeater.cc:499] Master 127.1.170.62:44507 was elected leader, sending a full tablet report...
21:49:14.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:14.700275 2484 heartbeater.cc:646] Failed to heartbeat to 127.1.170.60:37453 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.1.170.60:37453: connect: Connection refused (error 111)
21:49:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:14.374035 2512 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:49:16.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:16.290637 2498 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.916s user 0.662s sys 1.150s
21:49:16.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:16.291255 2498 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.918s user 0.662s sys 1.150s
21:49:16.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:15.786075 2510 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 2498
21:49:16.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:16.292810 2514 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:49:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:16.296617 2513 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1922 milliseconds
21:49:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.296658 2498 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:49:16.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.298143 2498 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:49:16.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.301642 2498 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:49:16.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.303166 2498 hybrid_clock.cc:648] HybridClock initialized: now 1756849756303132 us; error 55 us; skew 500 ppm
21:49:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.304466 2498 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:49:16.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:16 dist-test-slave-q7qm krb5kdc[1716](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849756, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:49:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.311406 2498 init.cc:377] Logged in from keytab as kudu/127.1.170.62@KRBTEST.COM (short username kudu)
21:49:16.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.317551 2498 webserver.cc:480] Webserver started at http://127.1.170.62:39525/ using document root <none> and password file <none>
21:49:16.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.318678 2498 fs_manager.cc:362] Metadata directory not provided
21:49:16.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.318951 2498 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:49:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.327704 2498 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.007s sys 0.000s
21:49:16.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.332541 2535 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:49:16.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.333729 2498 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
21:49:16.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.334038 2498 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/wal
21:49:16.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5196da08cb5e4ed9af16b991714eedb8"
21:49:16.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:55 on dist-test-slave-q7qm"
21:49:16.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.336081 2498 fs_report.cc:389] FS layout report
21:49:16.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:49:16.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/wal
21:49:16.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/wal
21:49:16.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/data/data
21:49:16.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:49:16.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:49:16.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:49:16.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:49:16.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:49:16.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:49:16.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:49:16.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:49:16.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:49:16.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:49:16.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:49:16.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.414404 2498 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:49:16.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.420727 2498 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:49:16.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.421245 2498 kserver.cc:163] Server-wide thread pool size limit: 3276
21:49:16.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.493219 2498 rpc_server.cc:307] RPC server started. Bound to: 127.1.170.62:44507
21:49:16.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.493408 2588 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.170.62:44507 every 8 connection(s)
21:49:16.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.496079 2498 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-0/data/info.pb
21:49:16.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.506155 1704 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 2498
21:49:16.506 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:49:16.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.506848 2589 sys_catalog.cc:263] Verifying existing consensus state
21:49:16.507 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.170.60:37453
21:49:16.507 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
21:49:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.508945 1704 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:49:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:49:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/wal
21:49:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/data
21:49:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:49:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:49:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:49:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:49:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:49:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:49:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/logs
21:49:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/data/info.pb
21:49:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:49:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:49:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:49:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:49:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:49:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:49:16.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:49:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:49:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:49:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:49:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:49:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:49:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.170.60:37453
21:49:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.170.60
21:49:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33881
21:49:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab
21:49:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.170.60
21:49:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:49:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:49:16.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:49:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:49:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.170.20:37829
21:49:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:49:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:49:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:49:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:49:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.170.62:44507,127.1.170.61:38649,127.1.170.60:37453
21:49:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:49:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41107/jwks.json
21:49:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:49:16.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:49:16.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.517055 2589 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8: Bootstrap starting.
21:49:16.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.572101 2589 log.cc:826] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8: Log is configured to *not* fsync() on all Append() calls
21:49:16.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.593046 2589 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:49:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.594079 2589 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8: Bootstrap complete.
21:49:16.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.637061 2589 raft_consensus.cc:357] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } }
21:49:16.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.641057 2589 raft_consensus.cc:738] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5196da08cb5e4ed9af16b991714eedb8, State: Initialized, Role: FOLLOWER
21:49:16.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.642297 2589 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } }
21:49:16.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.645444 2594 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } } }
21:49:16.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.646095 2594 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [sys.catalog]: This master's current role is: FOLLOWER
21:49:16.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.647944 2589 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [sys.catalog]: configured and running, proceeding with master startup.
21:49:16.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.700698 2605 catalog_manager.cc:1261] Loaded cluster ID: fabec8501a814001a48e777c37f96b91
21:49:16.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.701292 2605 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8: loading cluster ID for follower catalog manager: success
21:49:16.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.709867 2605 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8: acquiring CA information for follower catalog manager: success
21:49:16.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.720120 2605 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:49:16.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:16 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849756, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.62@KRBTEST.COM for kudu/127.1.170.62@KRBTEST.COM
21:49:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:16 dist-test-slave-q7qm krb5kdc[1716](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849756, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:49:16.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.741952 2070 init.cc:295] Successfully reacquired a new kerberos TGT
21:49:16.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.781423 2488 heartbeater.cc:344] Connected to a master server at 127.1.170.62:44507
21:49:16.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.782008 2488 heartbeater.cc:499] Master 127.1.170.62:44507 was elected leader, sending a full tablet report...
21:49:16.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.772877 2498 master_runner.cc:186] Error getting master registration for 127.1.170.61:38649: Network error: Client connection negotiation failed: client connection to 127.1.170.61:38649: connect: Connection refused (error 111),
21:49:16.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.810794 2498 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:49:16.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.813584 2554 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "62f724de78e74385a721887545e100f3" instance_seqno: 1756849753564181) as {username='kudu', principal='kudu/127.1.170.3@KRBTEST.COM'} at 127.1.170.3:47011; Asking this server to re-register.
21:49:16.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.816341 2488 heartbeater.cc:461] Registering TS with master...
21:49:16.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:16.821763 2554 ts_manager.cc:194] Registered new tserver with Master: 62f724de78e74385a721887545e100f3 (127.1.170.3:39057)
21:49:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:17.034145 2591 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:49:17.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:17.034968 2591 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:49:17.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:17.036139 2591 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:49:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:17.089712 2341 heartbeater.cc:344] Connected to a master server at 127.1.170.62:44507
21:49:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:17.092449 2554 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "d700b2d5bdb54547bd82dfc56a2b598f" instance_seqno: 1756849750824258) as {username='kudu', principal='kudu/127.1.170.2@KRBTEST.COM'} at 127.1.170.2:44461; Asking this server to re-register.
21:49:17.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:17.093966 2341 heartbeater.cc:461] Registering TS with master...
21:49:17.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:17.098122 2591 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:49:17.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:17.098202 2554 ts_manager.cc:194] Registered new tserver with Master: d700b2d5bdb54547bd82dfc56a2b598f (127.1.170.2:33647)
21:49:17.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:17.098454 2591 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:49:17.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:17.098917 2591 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:49:17.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:17.099248 2591 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:49:17.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:17.099602 2591 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:49:17.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:17.099912 2591 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:49:17.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:17.100124 2591 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41107/jwks.json
21:49:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:17.142644 2591 master_runner.cc:386] Master server non-default flags:
21:49:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:49:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.170.20:37829
21:49:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:49:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:49:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/data
21:49:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/wal
21:49:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:49:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:49:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.170.62:44507,127.1.170.61:38649,127.1.170.60:37453
21:49:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab
21:49:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.170.60
21:49:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:49:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:49:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:49:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.170.60:37453
21:49:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:49:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:49:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:49:17.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41107/jwks.json
21:49:17.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:49:17.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:49:17.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:49:17.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:49:17.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:49:17.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/data/info.pb
21:49:17.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.170.60
21:49:17.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33881
21:49:17.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:49:17.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:49:17.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:49:17.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:49:17.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:49:17.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/logs
21:49:17.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:49:17.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:49:17.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:49:17.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:49:17.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:49:17.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:49:17.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:49:17.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:49:17.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:49:17.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:49:17.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:17.144025 2591 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:49:17.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:17.145447 2591 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:49:17.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:17.158172 2619 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:49:17.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:17.419277 2193 heartbeater.cc:344] Connected to a master server at 127.1.170.62:44507
21:49:17.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:17.424965 2554 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "8abe630437664f39816cac5f99c5c6cf" instance_seqno: 1756849747911477) as {username='kudu', principal='kudu/127.1.170.1@KRBTEST.COM'} at 127.1.170.1:49461; Asking this server to re-register.
21:49:17.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:17.429203 2193 heartbeater.cc:461] Registering TS with master...
21:49:17.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:17.434105 2554 ts_manager.cc:194] Registered new tserver with Master: 8abe630437664f39816cac5f99c5c6cf (127.1.170.1:43877)
21:49:18.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:18.280694 2631 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:49:18.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:18.282042 2631 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } }
21:49:18.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:18.300176 2541 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.170.61:38649: connect: Connection refused (error 111)
21:49:18.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:18.302320 2631 leader_election.cc:290] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 680c50d5ebd8490a8173e465e7c7b077 (127.1.170.61:38649), 7cf4756128994aa38141b5167adb0d27 (127.1.170.60:37453)
21:49:18.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:18.318954 2541 leader_election.cc:336] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 680c50d5ebd8490a8173e465e7c7b077 (127.1.170.61:38649): Network error: Client connection negotiation failed: client connection to 127.1.170.61:38649: connect: Connection refused (error 111)
21:49:18.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:18.326669 2541 leader_election.cc:336] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 7cf4756128994aa38141b5167adb0d27 (127.1.170.60:37453): Network error: Client connection negotiation failed: client connection to 127.1.170.60:37453: connect: Connection refused (error 111)
21:49:18.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:18.328224 2541 leader_election.cc:304] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5196da08cb5e4ed9af16b991714eedb8; no voters: 680c50d5ebd8490a8173e465e7c7b077, 7cf4756128994aa38141b5167adb0d27
21:49:18.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:18.330996 2631 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
21:49:18.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:18.559947 2618 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 2591
21:49:18.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:18.984614 2618 kernel_stack_watchdog.cc:198] Thread 2591 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
21:49:18.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:49:18.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:49:18.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:49:18.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:49:18.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:18.988282 2622 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:49:18.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:18.986218 2591 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.826s user 0.400s sys 0.831s
21:49:18.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:18.989413 2591 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.831s user 0.400s sys 0.831s
21:49:18.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:17.159358 2620 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:49:18.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:18.995637 2621 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1835 milliseconds
21:49:18.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:18.995904 2591 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:49:18.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:18.998113 2591 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:49:19.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.000814 2591 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:49:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.002171 2591 hybrid_clock.cc:648] HybridClock initialized: now 1756849759002120 us; error 66 us; skew 500 ppm
21:49:19.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.003086 2591 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:49:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:19 dist-test-slave-q7qm krb5kdc[1716](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849759, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:49:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.009375 2591 init.cc:377] Logged in from keytab as kudu/127.1.170.60@KRBTEST.COM (short username kudu)
21:49:19.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.016688 2591 webserver.cc:480] Webserver started at http://127.1.170.60:33881/ using document root <none> and password file <none>
21:49:19.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.017762 2591 fs_manager.cc:362] Metadata directory not provided
21:49:19.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.017992 2591 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:49:19.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.026814 2591 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.000s
21:49:19.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.032225 2643 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:49:19.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.033524 2591 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
21:49:19.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.033846 2591 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/wal
21:49:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7cf4756128994aa38141b5167adb0d27"
21:49:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:49:01 on dist-test-slave-q7qm"
21:49:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.035919 2591 fs_report.cc:389] FS layout report
21:49:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:49:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/wal
21:49:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/wal
21:49:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/data/data
21:49:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:49:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:49:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:49:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:49:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:49:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:49:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:49:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:49:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:49:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:49:19.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:49:19.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.091921 2591 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:49:19.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.102025 2591 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:49:19.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.102509 2591 kserver.cc:163] Server-wide thread pool size limit: 3276
21:49:19.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.203938 2591 rpc_server.cc:307] RPC server started. Bound to: 127.1.170.60:37453
21:49:19.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.204193 2696 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.170.60:37453 every 8 connection(s)
21:49:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.206938 2591 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-2/data/info.pb
21:49:19.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.214669 1704 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 2591
21:49:19.216 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:49:19.216 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.170.61:38649
21:49:19.216 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
21:49:19.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.218762 1704 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:49:19.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu
21:49:19.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/wal
21:49:19.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/data
21:49:19.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:49:19.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:49:19.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:49:19.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:49:19.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:49:19.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:49:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/logs
21:49:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/data/info.pb
21:49:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:49:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:49:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:49:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:49:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:49:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:49:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:49:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:49:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:49:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:49:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:49:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:49:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.170.61:38649
21:49:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.170.61
21:49:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40769
21:49:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab
21:49:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.170.61
21:49:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:49:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:49:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:49:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:49:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.170.20:37829
21:49:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:49:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:49:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:49:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:49:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.170.62:44507,127.1.170.61:38649,127.1.170.60:37453
21:49:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:49:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41107/jwks.json
21:49:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:49:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:49:19.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.221328 2697 sys_catalog.cc:263] Verifying existing consensus state
21:49:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.230085 2697 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27: Bootstrap starting.
21:49:19.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.272419 2697 log.cc:826] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27: Log is configured to *not* fsync() on all Append() calls
21:49:19.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.293516 2697 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:49:19.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.294565 2697 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27: Bootstrap complete.
21:49:19.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.322986 2697 raft_consensus.cc:357] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } }
21:49:19.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.326951 2697 raft_consensus.cc:738] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7cf4756128994aa38141b5167adb0d27, State: Initialized, Role: FOLLOWER
21:49:19.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.328382 2697 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } }
21:49:19.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.336138 2701 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } } }
21:49:19.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.337320 2701 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [sys.catalog]: This master's current role is: FOLLOWER
21:49:19.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.338541 2697 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [sys.catalog]: configured and running, proceeding with master startup.
21:49:19.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.414774 2712 catalog_manager.cc:1261] Loaded cluster ID: fabec8501a814001a48e777c37f96b91
21:49:19.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.415151 2712 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27: loading cluster ID for follower catalog manager: success
21:49:19.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.424073 2712 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27: acquiring CA information for follower catalog manager: success
21:49:19.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.432113 2712 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:49:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:19 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849759, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.60@KRBTEST.COM for kudu/127.1.170.62@KRBTEST.COM
21:49:19.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.464133 2188 heartbeater.cc:344] Connected to a master server at 127.1.170.60:37453
21:49:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.471556 2662 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "8abe630437664f39816cac5f99c5c6cf" instance_seqno: 1756849747911477) as {username='kudu', principal='kudu/127.1.170.1@KRBTEST.COM'} at 127.1.170.1:33585; Asking this server to re-register.
21:49:19.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.473435 2188 heartbeater.cc:461] Registering TS with master...
21:49:19.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.478072 2661 ts_manager.cc:194] Registered new tserver with Master: 8abe630437664f39816cac5f99c5c6cf (127.1.170.1:43877)
21:49:19.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.488965 2591 master_runner.cc:186] Error getting master registration for 127.1.170.61:38649: Network error: Client connection negotiation failed: client connection to 127.1.170.61:38649: connect: Connection refused (error 111),
21:49:19.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.563763 2591 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:49:19.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:19.821805 2699 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:49:19.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:19.822602 2699 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:49:19.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:19.823735 2699 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:49:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.849128 2484 heartbeater.cc:344] Connected to a master server at 127.1.170.60:37453
21:49:19.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.852476 2661 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "62f724de78e74385a721887545e100f3" instance_seqno: 1756849753564181) as {username='kudu', principal='kudu/127.1.170.3@KRBTEST.COM'} at 127.1.170.3:58965; Asking this server to re-register.
21:49:19.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.854609 2484 heartbeater.cc:461] Registering TS with master...
21:49:19.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.857926 2661 ts_manager.cc:194] Registered new tserver with Master: 62f724de78e74385a721887545e100f3 (127.1.170.3:39057)
21:49:19.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:19.902603 2699 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:49:19.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:19.902947 2699 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:49:19.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:19.903472 2699 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:49:19.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:19.903861 2699 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:49:19.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:19.904201 2699 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:49:19.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:19.904532 2699 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:49:19.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:19.904794 2699 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41107/jwks.json
21:49:19.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.973886 2699 master_runner.cc:386] Master server non-default flags:
21:49:19.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:49:19.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.170.20:37829
21:49:19.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:49:19.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:49:19.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/data
21:49:19.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/wal
21:49:19.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:49:19.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:49:19.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.170.62:44507,127.1.170.61:38649,127.1.170.60:37453
21:49:19.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/krb5kdc/kudu.keytab
21:49:19.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.170.61
21:49:19.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:49:19.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:49:19.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:49:19.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.170.61:38649
21:49:19.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:49:19.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:49:19.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:49:19.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41107/jwks.json
21:49:19.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:49:19.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:49:19.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:49:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:49:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:49:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/data/info.pb
21:49:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.170.61
21:49:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40769
21:49:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:49:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:49:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:49:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:49:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:49:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/logs
21:49:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:49:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:49:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:49:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:49:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:49:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
21:49:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:49:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
21:49:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
21:49:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:49:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.975616 2699 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:49:19.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:19.977005 2699 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:49:19.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:19.993995 2726 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:49:20.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.177526 2337 heartbeater.cc:344] Connected to a master server at 127.1.170.60:37453
21:49:20.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.181051 2661 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "d700b2d5bdb54547bd82dfc56a2b598f" instance_seqno: 1756849750824258) as {username='kudu', principal='kudu/127.1.170.2@KRBTEST.COM'} at 127.1.170.2:58281; Asking this server to re-register.
21:49:20.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.184144 2337 heartbeater.cc:461] Registering TS with master...
21:49:20.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.189379 2661 ts_manager.cc:194] Registered new tserver with Master: d700b2d5bdb54547bd82dfc56a2b598f (127.1.170.2:33647)
21:49:20.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.219205 2735 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:49:20.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.223651 2735 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } }
21:49:20.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.243129 2735 leader_election.cc:290] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 680c50d5ebd8490a8173e465e7c7b077 (127.1.170.61:38649), 7cf4756128994aa38141b5167adb0d27 (127.1.170.60:37453)
21:49:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:20.275491 2541 leader_election.cc:336] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 680c50d5ebd8490a8173e465e7c7b077 (127.1.170.61:38649): Network error: Client connection negotiation failed: client connection to 127.1.170.61:38649: connect: Connection refused (error 111)
21:49:20.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.293358 2672 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5196da08cb5e4ed9af16b991714eedb8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "7cf4756128994aa38141b5167adb0d27" is_pre_election: true
21:49:20.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.294864 2672 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5196da08cb5e4ed9af16b991714eedb8 in term 1.
21:49:20.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.296756 2541 leader_election.cc:304] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5196da08cb5e4ed9af16b991714eedb8, 7cf4756128994aa38141b5167adb0d27; no voters: 680c50d5ebd8490a8173e465e7c7b077
21:49:20.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.298167 2735 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 1 FOLLOWER]: Leader pre-election won for term 2
21:49:20.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.298527 2735 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:49:20.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.298854 2735 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 1 FOLLOWER]: Advancing to term 2
21:49:20.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.339036 2735 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } }
21:49:20.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:20.353412 2541 leader_election.cc:336] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 680c50d5ebd8490a8173e465e7c7b077 (127.1.170.61:38649): Network error: Client connection negotiation failed: client connection to 127.1.170.61:38649: connect: Connection refused (error 111)
21:49:20.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.356406 2672 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5196da08cb5e4ed9af16b991714eedb8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "7cf4756128994aa38141b5167adb0d27"
21:49:20.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.357250 2672 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [term 1 FOLLOWER]: Advancing to term 2
21:49:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.378774 2735 leader_election.cc:290] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [CANDIDATE]: Term 2 election: Requested vote from peers 680c50d5ebd8490a8173e465e7c7b077 (127.1.170.61:38649), 7cf4756128994aa38141b5167adb0d27 (127.1.170.60:37453)
21:49:20.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.411185 2672 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5196da08cb5e4ed9af16b991714eedb8 in term 2.
21:49:20.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.418072 2541 leader_election.cc:304] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5196da08cb5e4ed9af16b991714eedb8, 7cf4756128994aa38141b5167adb0d27; no voters: 680c50d5ebd8490a8173e465e7c7b077
21:49:20.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.420044 2735 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 2 FOLLOWER]: Leader election won for term 2
21:49:20.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.442075 2735 raft_consensus.cc:695] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [term 2 LEADER]: Becoming Leader. State: Replica: 5196da08cb5e4ed9af16b991714eedb8, State: Running, Role: LEADER
21:49:20.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.444069 2735 consensus_queue.cc:237] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } }
21:49:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.479909 2739 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5196da08cb5e4ed9af16b991714eedb8. Latest consensus state: current_term: 2 leader_uuid: "5196da08cb5e4ed9af16b991714eedb8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } } }
21:49:20.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.480872 2739 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [sys.catalog]: This master's current role is: LEADER
21:49:20.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.491485 2742 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
21:49:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.527501 2742 catalog_manager.cc:1486] Initializing Kudu cluster ID...
21:49:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.544198 2742 catalog_manager.cc:1261] Loaded cluster ID: fabec8501a814001a48e777c37f96b91
21:49:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.544657 2742 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
21:49:20.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.557752 2742 catalog_manager.cc:1506] Loading token signing keys...
21:49:20.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.574573 2742 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8: Loaded TSK: 0
21:49:20.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.576548 2742 catalog_manager.cc:1516] Initializing in-progress tserver states...
21:49:20.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:20 dist-test-slave-q7qm krb5kdc[1716](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849760, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:49:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.606709 2220 init.cc:295] Successfully reacquired a new kerberos TGT
21:49:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:20.787730 1689 debug-util.cc:398] Leaking SignalData structure 0x7b08000b44a0 after lost signal to thread 1564
21:49:20.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:20.789525 1689 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac600 after lost signal to thread 1692
21:49:20.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.850462 2672 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [term 2 FOLLOWER]: Refusing update from remote peer 5196da08cb5e4ed9af16b991714eedb8: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
21:49:20.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.870496 2739 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
21:49:20.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:20.902725 2541 consensus_peers.cc:489] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 -> Peer 680c50d5ebd8490a8173e465e7c7b077 (127.1.170.61:38649): Couldn't send request to peer 680c50d5ebd8490a8173e465e7c7b077. Status: Network error: Client connection negotiation failed: client connection to 127.1.170.61:38649: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:49:20.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.906350 2744 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5196da08cb5e4ed9af16b991714eedb8. Latest consensus state: current_term: 2 leader_uuid: "5196da08cb5e4ed9af16b991714eedb8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } } }
21:49:20.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.910463 2744 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [sys.catalog]: This master's current role is: FOLLOWER
21:49:20.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.913362 2744 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "5196da08cb5e4ed9af16b991714eedb8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } } }
21:49:20.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.914587 2744 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7cf4756128994aa38141b5167adb0d27 [sys.catalog]: This master's current role is: FOLLOWER
21:49:20.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.915459 2735 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "5196da08cb5e4ed9af16b991714eedb8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } } }
21:49:20.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:20.916374 2735 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [sys.catalog]: This master's current role is: LEADER
21:49:21.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:21.820459 2498 master_runner.cc:186] Error getting master registration for 127.1.170.61:38649: Network error: Client connection negotiation failed: client connection to 127.1.170.61:38649: connect: Connection refused (error 111),
21:49:21.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:21.820958 2498 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:49:21.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:21.863658 2488 heartbeater.cc:499] Master 127.1.170.62:44507 was elected leader, sending a full tablet report...
21:49:22.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:22.129876 2341 heartbeater.cc:499] Master 127.1.170.62:44507 was elected leader, sending a full tablet report...
21:49:22.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:22.483124 2193 heartbeater.cc:499] Master 127.1.170.62:44507 was elected leader, sending a full tablet report...
21:49:23.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:23 dist-test-slave-q7qm krb5kdc[1716](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849763, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:49:23.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.253978 2367 init.cc:295] Successfully reacquired a new kerberos TGT
21:49:23.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:23.349750 2541 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.170.61:38649: connect: Connection refused (error 111) [suppressed 8 similar messages]
21:49:23.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:23.362540 2541 consensus_peers.cc:489] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 -> Peer 680c50d5ebd8490a8173e465e7c7b077 (127.1.170.61:38649): Couldn't send request to peer 680c50d5ebd8490a8173e465e7c7b077. Status: Network error: Client connection negotiation failed: client connection to 127.1.170.61:38649: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
21:49:23.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:21.395749 2725 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 2699
21:49:23.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:23.530547 2725 kernel_stack_watchdog.cc:198] Thread 2699 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 397ms:
21:49:23.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:49:23.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:49:23.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:49:23.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:49:23.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:23.531527 2699 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 3.535s user 1.237s sys 1.822s
21:49:23.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:23.531941 2699 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 3.535s user 1.237s sys 1.822s
21:49:23.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:23.533392 2728 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3534 milliseconds
21:49:23.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:19.996822 2727 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:49:23.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:23.534461 2729 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:49:23.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.535467 2699 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:49:23.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.540059 2699 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:49:23.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.548051 2699 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:49:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.551789 2699 hybrid_clock.cc:648] HybridClock initialized: now 1756849763551740 us; error 49 us; skew 500 ppm
21:49:23.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.552891 2699 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:49:23.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:23 dist-test-slave-q7qm krb5kdc[1716](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849763, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:49:23.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.561297 2699 init.cc:377] Logged in from keytab as kudu/127.1.170.61@KRBTEST.COM (short username kudu)
21:49:23.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.568080 2699 webserver.cc:480] Webserver started at http://127.1.170.61:40769/ using document root <none> and password file <none>
21:49:23.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.569252 2699 fs_manager.cc:362] Metadata directory not provided
21:49:23.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.569538 2699 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:49:23.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.578866 2699 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.005s sys 0.001s
21:49:23.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.584400 2770 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:49:23.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.585706 2699 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.000s sys 0.004s
21:49:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.586076 2699 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/data,/tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/wal
21:49:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "680c50d5ebd8490a8173e465e7c7b077"
21:49:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 21:48:58 on dist-test-slave-q7qm"
21:49:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.588296 2699 fs_report.cc:389] FS layout report
21:49:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:49:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/wal
21:49:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/wal
21:49:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/data/data
21:49:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:49:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:49:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:49:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:49:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:49:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:49:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:49:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:49:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:49:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:49:23.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:49:23.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.697973 2699 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:49:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.705925 2699 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:49:23.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.706904 2699 kserver.cc:163] Server-wide thread pool size limit: 3276
21:49:23.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.932405 2699 rpc_server.cc:307] RPC server started. Bound to: 127.1.170.61:38649
21:49:23.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.932626 2824 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.170.61:38649 every 8 connection(s)
21:49:23.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.936568 2699 server_base.cc:1179] Dumped server information to /tmp/dist-test-tasktmwPRy/test-tmp/mini-kudu-cluster3204059251090892781/master-1/data/info.pb
21:49:23.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.944854 1704 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu as pid 2699
21:49:23.946 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:49:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.975270 2827 sys_catalog.cc:263] Verifying existing consensus state
21:49:23.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:23.992815 2827 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077: Bootstrap starting.
21:49:24.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:24 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849756, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.62@KRBTEST.COM for kudu/127.1.170.61@KRBTEST.COM
21:49:24.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:24 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849763, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.3@KRBTEST.COM for kudu/127.1.170.61@KRBTEST.COM
21:49:24.067 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.170.62:44507(127.1.170.62:44507)] connecting to peer
21:49:24.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.096433 2485 heartbeater.cc:344] Connected to a master server at 127.1.170.61:38649
21:49:24.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.128484 2827 log.cc:826] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077: Log is configured to *not* fsync() on all Append() calls
21:49:24.157 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.170.60:37453(127.1.170.60:37453)] connecting to peer
21:49:24.161 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.170.61:38649(127.1.170.61:38649)] connecting to peer
21:49:24.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.195968 2827 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:49:24.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.197269 2827 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077: Bootstrap complete.
21:49:24.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.232146 2827 raft_consensus.cc:357] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } }
21:49:24.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.236279 2827 raft_consensus.cc:738] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 680c50d5ebd8490a8173e465e7c7b077, State: Initialized, Role: FOLLOWER
21:49:24.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.237579 2827 consensus_queue.cc:260] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } }
21:49:24.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.243731 2827 sys_catalog.cc:564] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [sys.catalog]: configured and running, proceeding with master startup.
21:49:24.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 21:49:24 dist-test-slave-q7qm krb5kdc[1716](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756849760, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.170.2@KRBTEST.COM for kudu/127.1.170.61@KRBTEST.COM
21:49:24.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.246973 2797 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [term 1 FOLLOWER]: Advancing to term 2
21:49:24.261 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.1.170.60:37453(127.1.170.60:37453)] Successfully connected to peer
21:49:24.264 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.1.170.62:44507(127.1.170.62:44507)] Successfully connected to peer
21:49:24.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.291435 2338 heartbeater.cc:344] Connected to a master server at 127.1.170.61:38649
21:49:24.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.241608 2837 sys_catalog.cc:455] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } } }
21:49:24.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.301110 2837 sys_catalog.cc:458] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [sys.catalog]: This master's current role is: FOLLOWER
21:49:24.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.356336 2846 sys_catalog.cc:455] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "5196da08cb5e4ed9af16b991714eedb8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } } }
21:49:24.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.357637 2846 sys_catalog.cc:458] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [sys.catalog]: This master's current role is: FOLLOWER
21:49:24.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.364033 2837 sys_catalog.cc:455] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5196da08cb5e4ed9af16b991714eedb8. Latest consensus state: current_term: 2 leader_uuid: "5196da08cb5e4ed9af16b991714eedb8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } } }
21:49:24.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.364861 2837 sys_catalog.cc:458] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077 [sys.catalog]: This master's current role is: FOLLOWER
21:49:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.410745 2811 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "5196da08cb5e4ed9af16b991714eedb8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5196da08cb5e4ed9af16b991714eedb8" member_type: VOTER last_known_addr { host: "127.1.170.62" port: 44507 } } peers { permanent_uuid: "680c50d5ebd8490a8173e465e7c7b077" member_type: VOTER last_known_addr { host: "127.1.170.61" port: 38649 } } peers { permanent_uuid: "7cf4756128994aa38141b5167adb0d27" member_type: VOTER last_known_addr { host: "127.1.170.60" port: 37453 } } }
21:49:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.413008 2854 catalog_manager.cc:1261] Loaded cluster ID: fabec8501a814001a48e777c37f96b91
21:49:24.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.413285 2854 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077: loading cluster ID for follower catalog manager: success
21:49:24.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.415817 2811 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5196da08cb5e4ed9af16b991714eedb8 [sys.catalog]: This master's current role is: LEADER
21:49:24.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.422585 2854 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077: acquiring CA information for follower catalog manager: success
21:49:24.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.428553 2854 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 680c50d5ebd8490a8173e465e7c7b077: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:49:24.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:24.483063 1210 debug-util.cc:398] Leaking SignalData structure 0x7b0800097440 after lost signal to thread 1141
21:49:24.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:24.483814 1210 debug-util.cc:398] Leaking SignalData structure 0x7b08000b68c0 after lost signal to thread 1213
21:49:24.487 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.1.170.61:38649(127.1.170.61:38649)] Successfully connected to peer
21:49:24.521 [WARN - kudu-nio-0] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
21:49:24.525 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.170.62:44507(127.1.170.62:44507)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
21:49:24.527 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.170.60:37453(127.1.170.60:37453)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
21:49:24.528 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.170.60:37453: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
21:49:24.530 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.1.170.60:37453(127.1.170.60:37453)] handling channelInactive
21:49:24.531 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.170.62:44507: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
21:49:24.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:24.531091 2839 negotiation.cc:337] Failed RPC negotiation. Trace:
21:49:24.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.247748 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:35248 (local address 127.1.170.60:37453)
21:49:24.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.311353 (+ 63605us) server_negotiation.cc:207] Beginning negotiation
21:49:24.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.311416 (+ 63us) server_negotiation.cc:400] Waiting for connection header
21:49:24.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.375802 (+ 64386us) server_negotiation.cc:408] Connection header received
21:49:24.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.487773 (+111971us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:49:24.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.487807 (+ 34us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:49:24.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.488054 (+ 247us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:49:24.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.488258 (+ 204us) server_negotiation.cc:227] Negotiated authn=SASL
21:49:24.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.529622 (+ 41364us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:35248: BlockingRecv error: recv got EOF from 127.0.0.1:35248 (error 108)
21:49:24.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":62211,"thread_start_us":63178,"threads_started":1}
21:49:24.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:24.533980 2841 negotiation.cc:337] Failed RPC negotiation. Trace:
21:49:24.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.255760 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:41088 (local address 127.1.170.62:44507)
21:49:24.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.280482 (+ 24722us) server_negotiation.cc:207] Beginning negotiation
21:49:24.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.280517 (+ 35us) server_negotiation.cc:400] Waiting for connection header
21:49:24.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.375783 (+ 95266us) server_negotiation.cc:408] Connection header received
21:49:24.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.485092 (+109309us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:49:24.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.485134 (+ 42us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:49:24.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.485389 (+ 255us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:49:24.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.485571 (+ 182us) server_negotiation.cc:227] Negotiated authn=SASL
21:49:24.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.532332 (+ 46761us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:41088: BlockingRecv error: recv got EOF from 127.0.0.1:41088 (error 108)
21:49:24.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":23979,"thread_start_us":67284,"threads_started":1}
21:49:24.536 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.170.61:38649(127.1.170.61:38649)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
21:49:24.536 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.170.61:38649: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
21:49:24.542 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=590), Traces: [0ms] refreshing cache from master, [88ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.170.62:44507, [197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.170.60:37453, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.170.61:38649, [567ms] Sub RPC ConnectToMaster: received response from server master-127.1.170.60:37453: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [566ms] Sub RPC ConnectToMaster: received response from server master-127.1.170.62:44507: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [576ms] Sub RPC ConnectToMaster: received response from server master-127.1.170.61:38649: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.1.170.62:44507,127.1.170.60:37453,127.1.170.61:38649). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) [main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) [main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) [async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) [main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) [main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) [main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) [netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.1.170.62:44507,127.1.170.60:37453,127.1.170.61:38649). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
21:49:24.548 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=597), Traces: [0ms] refreshing cache from master, [88ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.170.62:44507, [197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.170.60:37453, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.170.61:38649, [567ms] Sub RPC ConnectToMaster: received response from server master-127.1.170.60:37453: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [566ms] Sub RPC ConnectToMaster: received response from server master-127.1.170.62:44507: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [576ms] Sub RPC ConnectToMaster: received response from server master-127.1.170.61:38649: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) with no deferred attached!
21:49:24.552 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.170.61:38649(127.1.170.61:38649)] disconnecting while in state TERMINATED
21:49:24.553 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.170.62:44507(127.1.170.62:44507)] disconnecting while in state TERMINATED
21:49:24.553 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.170.60:37453(127.1.170.60:37453)] disconnecting while in state TERMINATED
21:49:24.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:24.560432 2831 negotiation.cc:337] Failed RPC negotiation. Trace:
21:49:24.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.268197 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:42128 (local address 127.1.170.61:38649)
21:49:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.269130 (+ 933us) server_negotiation.cc:207] Beginning negotiation
21:49:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.269167 (+ 37us) server_negotiation.cc:400] Waiting for connection header
21:49:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.487813 (+218646us) server_negotiation.cc:408] Connection header received
21:49:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.497704 (+ 9891us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:49:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.497740 (+ 36us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:49:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.498135 (+ 395us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:49:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.498334 (+ 199us) server_negotiation.cc:227] Negotiated authn=SASL
21:49:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 21:49:24.558596 (+ 60262us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:42128: BlockingRecv error: recv got EOF from 127.0.0.1:42128 (error 108)
21:49:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":148}
21:49:24.562 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.1.170.62:44507(127.1.170.62:44507)] handling channelInactive
21:49:24.564 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:49:24.577 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:49:24.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.577334 2189 heartbeater.cc:344] Connected to a master server at 127.1.170.61:38649
21:49:24.579 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.1.170.61:38649(127.1.170.61:38649)] handling channelInactive
21:49:24.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.584281 2787 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "8abe630437664f39816cac5f99c5c6cf" instance_seqno: 1756849747911477) as {username='kudu', principal='kudu/127.1.170.1@KRBTEST.COM'} at 127.1.170.1:33251; Asking this server to re-register.
21:49:24.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.585801 2189 heartbeater.cc:461] Registering TS with master...
21:49:24.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.589491 2787 ts_manager.cc:194] Registered new tserver with Master: 8abe630437664f39816cac5f99c5c6cf (127.1.170.1:43877)
21:49:24.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.643486 1704 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 2047
21:49:24.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.733630 1704 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 2202
21:49:24.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.811805 1704 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 2351
21:49:24.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.881826 1704 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 2498
21:49:24.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 21:49:24.941953 2820 messenger.cc:191] Invalid argument: socket address is ambiguous: no data on pending connections for acceptor pool at 127.1.170.61:38649
21:49:24.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:24.967486 1704 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 2699
21:49:25.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:25.014590 1704 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 2591
21:49:26.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T21:49:26Z chronyd exiting
21:49:27.424 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:49:27.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:27.426700 979 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 1274
21:49:27.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:27.454602 979 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 1422
21:49:27.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:27.484797 979 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 1563
21:49:27.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:27.520108 979 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 991
21:49:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:27.553995 979 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 1061
21:49:27.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 21:49:27.592265 979 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktmwPRy/build/tsan/bin/kudu with pid 1136
21:49:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T21:49:28Z chronyd exiting
21:49:29.819 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.1.170.62:44507,127.1.170.60:37453,127.1.170.61:38649). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.1.170.62:44507,127.1.170.60:37453,127.1.170.61:38649). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
21:49:30.316 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
21:49:30.316 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.1.170.62:44507,127.1.170.60:37453,127.1.170.61:38649). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.1.170.62:44507,127.1.170.60:37453,127.1.170.61:38649). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more