22:03:55.519 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
22:03:55.519 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task3DGRBN/test-tmp
22:03:55.520 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
22:03:55.520 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task3DGRBN/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
22:03:55.521 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
22:03:55.846 [DEBUG - main] (MiniKuduCluster.java:195) Response:
22:03:55.846 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
22:03:55.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:55.850021 24828 mini_oidc.cc:150] Starting JWKS server
22:03:55.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:55.853459 24828 webserver.cc:480] Webserver started at http://127.0.0.1:34301/ using document root <none> and password file <none>
22:03:55.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:55.853904 24828 mini_oidc.cc:186] Starting OIDC Discovery server
22:03:55.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:55.855594 24828 webserver.cc:480] Webserver started at http://127.0.0.1:44577/ using document root <none> and password file <none>
22:03:55.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T22:03:55Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
22:03:55.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T22:03:55Z Disabled control of system clock
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:55.919936 24828 external_mini_cluster.cc:1366] Running /tmp/dist-test-task3DGRBN/build/tsan/bin/kudu
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3DGRBN/build/tsan/bin/kudu
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-0/wal
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-0/data
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-0/logs
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-0/data/info.pb
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.63.62:41701
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.63.62
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.63.20:39889
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.63.62:41701,127.24.63.61:38085,127.24.63.60:44011
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:03:55.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34301/jwks.json with env {}
22:03:56.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:56.220165 24840 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:03:56.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:56.220733 24840 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:03:56.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:56.221213 24840 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:03:56.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:56.253268 24840 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:03:56.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:56.253623 24840 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:03:56.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:56.253901 24840 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:03:56.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:56.254127 24840 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:03:56.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:56.254312 24840 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:03:56.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:56.254472 24840 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34301/jwks.json
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:56.291982 24840 master_runner.cc:386] Master server non-default flags:
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.63.20:39889
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-0/data
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-0/wal
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.63.62:41701,127.24.63.61:38085,127.24.63.60:44011
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.63.62:41701
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34301/jwks.json
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-0/data/info.pb
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.63.62
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-0/logs
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:03:56.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:03:56.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
22:03:56.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
22:03:56.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:03:56.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
22:03:56.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
22:03:56.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:03:56.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:56.293282 24840 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:03:56.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:56.294968 24840 file_cache.cc:492] Constructed file cache file cache with capacity 419430
22:03:56.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:56.306191 24846 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:03:57.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:56.311506 24849 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:03:57.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:56.307312 24847 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:03:57.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:57.517367 24848 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1206 milliseconds
22:03:57.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.517488 24840 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
22:03:57.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.518761 24840 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:03:57.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.521955 24840 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:03:57.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.523432 24840 hybrid_clock.cc:648] HybridClock initialized: now 1756850637523381 us; error 73 us; skew 500 ppm
22:03:57.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.524238 24840 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:03:57.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.530987 24840 webserver.cc:480] Webserver started at http://127.24.63.62:40371/ using document root <none> and password file <none>
22:03:57.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.531903 24840 fs_manager.cc:362] Metadata directory not provided
22:03:57.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.532095 24840 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:03:57.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.532523 24840 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
22:03:57.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.537091 24840 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-0/data/instance:
22:03:57.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1e623a08940348cab7185f81195de1d6"
22:03:57.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 22:03:57 on dist-test-slave-53g9"
22:03:57.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.538262 24840 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-0/wal/instance:
22:03:57.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1e623a08940348cab7185f81195de1d6"
22:03:57.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 22:03:57 on dist-test-slave-53g9"
22:03:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.545925 24840 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
22:03:57.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.551836 24856 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:03:57.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.553028 24840 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.003s
22:03:57.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.553347 24840 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-0/data,/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-0/wal
22:03:57.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1e623a08940348cab7185f81195de1d6"
22:03:57.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 22:03:57 on dist-test-slave-53g9"
22:03:57.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.553674 24840 fs_report.cc:389] FS layout report
22:03:57.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:03:57.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-0/wal
22:03:57.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-0/wal
22:03:57.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-0/data/data
22:03:57.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:03:57.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:03:57.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:03:57.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:03:57.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:03:57.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:03:57.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:03:57.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:03:57.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:03:57.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:03:57.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:03:57.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.601938 24840 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
22:03:57.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.603477 24840 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:03:57.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.603899 24840 kserver.cc:163] Server-wide thread pool size limit: 3276
22:03:57.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.679412 24840 rpc_server.cc:307] RPC server started. Bound to: 127.24.63.62:41701
22:03:57.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.679476 24907 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.63.62:41701 every 8 connection(s)
22:03:57.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.682206 24840 server_base.cc:1179] Dumped server information to /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-0/data/info.pb
22:03:57.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.687448 24908 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:03:57.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.689698 24828 external_mini_cluster.cc:1428] Started /tmp/dist-test-task3DGRBN/build/tsan/bin/kudu as pid 24840
22:03:57.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.690174 24828 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-0/wal/instance
22:03:57.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.693992 24828 external_mini_cluster.cc:1366] Running /tmp/dist-test-task3DGRBN/build/tsan/bin/kudu
22:03:57.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3DGRBN/build/tsan/bin/kudu
22:03:57.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-1/wal
22:03:57.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-1/data
22:03:57.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:03:57.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:03:57.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:03:57.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:03:57.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:03:57.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:03:57.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-1/logs
22:03:57.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-1/data/info.pb
22:03:57.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:03:57.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:03:57.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:03:57.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:03:57.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:03:57.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:03:57.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:03:57.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:03:57.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:03:57.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:03:57.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:03:57.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:03:57.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.63.61:38085
22:03:57.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.63.61
22:03:57.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:03:57.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.63.20:39889
22:03:57.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:03:57.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:03:57.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:03:57.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:03:57.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.63.62:41701,127.24.63.61:38085,127.24.63.60:44011
22:03:57.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:03:57.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34301/jwks.json with env {}
22:03:57.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.699595 24908 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.63.62" port: 41701 } has no permanent_uuid. Determining permanent_uuid...
22:03:57.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.732169 24908 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } has no permanent_uuid. Determining permanent_uuid...
22:03:57.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:57.734601 24858 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.63.61:38085: connect: Connection refused (error 111)
22:03:57.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:57.738902 24908 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.63.61:38085: Network error: Client connection negotiation failed: client connection to 127.24.63.61:38085: connect: Connection refused (error 111)
22:03:57.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.788506 24908 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } attempt: 1
22:03:57.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:57.792497 24908 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.63.61:38085: Network error: Client connection negotiation failed: client connection to 127.24.63.61:38085: connect: Connection refused (error 111)
22:03:57.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.861037 24908 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } attempt: 2
22:03:57.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:57.865090 24908 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.63.61:38085: Network error: Client connection negotiation failed: client connection to 127.24.63.61:38085: connect: Connection refused (error 111)
22:03:57.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:57.956681 24908 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } attempt: 3
22:03:57.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:57.960749 24908 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.63.61:38085: Network error: Client connection negotiation failed: client connection to 127.24.63.61:38085: connect: Connection refused (error 111)
22:03:58.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:58.010532 24910 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:03:58.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:58.011093 24910 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:03:58.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:58.011579 24910 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:03:58.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:58.043310 24910 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:03:58.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:58.043623 24910 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:03:58.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:58.043835 24910 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:03:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:58.044039 24910 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:03:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:58.044230 24910 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:03:58.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:58.044379 24910 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34301/jwks.json
22:03:58.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:58.081363 24910 master_runner.cc:386] Master server non-default flags:
22:03:58.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:03:58.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.63.20:39889
22:03:58.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:03:58.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:03:58.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-1/data
22:03:58.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-1/wal
22:03:58.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:03:58.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.63.62:41701,127.24.63.61:38085,127.24.63.60:44011
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.63.61:38085
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34301/jwks.json
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-1/data/info.pb
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.63.61
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-1/logs
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:03:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:58.082718 24910 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:03:58.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:58.084372 24910 file_cache.cc:492] Constructed file cache file cache with capacity 419430
22:03:58.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:58.095737 24919 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:03:58.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:58.104269 24908 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } attempt: 4
22:03:58.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:58.111949 24908 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.63.61:38085: Network error: Client connection negotiation failed: client connection to 127.24.63.61:38085: connect: Connection refused (error 111)
22:03:58.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:58.411387 24908 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } attempt: 5
22:03:58.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:58.423362 24908 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.63.61:38085: Network error: Client connection negotiation failed: client connection to 127.24.63.61:38085: connect: Connection refused (error 111)
22:03:58.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:58.970994 24908 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } attempt: 6
22:03:58.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:58.999059 24908 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.63.61:38085: Network error: Client connection negotiation failed: client connection to 127.24.63.61:38085: connect: Connection refused (error 111)
22:03:59.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:59.500798 24918 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ec0 after lost signal to thread 24910
22:03:59.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:59.630703 24918 kernel_stack_watchdog.cc:198] Thread 24910 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
22:03:59.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
22:03:59.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
22:03:59.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
22:03:59.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
22:03:59.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:58.096297 24920 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:03:59.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:59.632244 24910 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.536s user 0.484s sys 1.052s
22:03:59.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:59.633034 24921 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1535 milliseconds
22:03:59.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:59.633307 24910 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.537s user 0.484s sys 1.052s
22:03:59.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.634593 24910 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
22:03:59.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:59.634651 24922 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:03:59.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.637938 24910 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:03:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.640507 24910 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:03:59.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.641884 24910 hybrid_clock.cc:648] HybridClock initialized: now 1756850639641838 us; error 58 us; skew 500 ppm
22:03:59.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.642741 24910 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:03:59.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.648936 24910 webserver.cc:480] Webserver started at http://127.24.63.61:39311/ using document root <none> and password file <none>
22:03:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.649926 24910 fs_manager.cc:362] Metadata directory not provided
22:03:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.650190 24910 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:03:59.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.650705 24910 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
22:03:59.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.655362 24910 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-1/data/instance:
22:03:59.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a0a91eb6a56442589425efb2c6316870"
22:03:59.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 22:03:59 on dist-test-slave-53g9"
22:03:59.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.656574 24910 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-1/wal/instance:
22:03:59.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a0a91eb6a56442589425efb2c6316870"
22:03:59.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 22:03:59 on dist-test-slave-53g9"
22:03:59.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.664054 24910 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.002s
22:03:59.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.669893 24931 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:03:59.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.670979 24910 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
22:03:59.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.671348 24910 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-1/data,/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-1/wal
22:03:59.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a0a91eb6a56442589425efb2c6316870"
22:03:59.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 22:03:59 on dist-test-slave-53g9"
22:03:59.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.671682 24910 fs_report.cc:389] FS layout report
22:03:59.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:03:59.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-1/wal
22:03:59.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-1/wal
22:03:59.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-1/data/data
22:03:59.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:03:59.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:03:59.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:03:59.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:03:59.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:03:59.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:03:59.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:03:59.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:03:59.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:03:59.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:03:59.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:03:59.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.722929 24910 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
22:03:59.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.724476 24910 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:03:59.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.724911 24910 kserver.cc:163] Server-wide thread pool size limit: 3276
22:03:59.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.796515 24910 rpc_server.cc:307] RPC server started. Bound to: 127.24.63.61:38085
22:03:59.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.796604 24982 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.63.61:38085 every 8 connection(s)
22:03:59.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.799206 24910 server_base.cc:1179] Dumped server information to /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-1/data/info.pb
22:03:59.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.804404 24983 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:03:59.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.805503 24828 external_mini_cluster.cc:1428] Started /tmp/dist-test-task3DGRBN/build/tsan/bin/kudu as pid 24910
22:03:59.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.805998 24828 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-1/wal/instance
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.809049 24828 external_mini_cluster.cc:1366] Running /tmp/dist-test-task3DGRBN/build/tsan/bin/kudu
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3DGRBN/build/tsan/bin/kudu
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-2/wal
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-2/data
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-2/logs
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-2/data/info.pb
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.63.60:44011
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.63.60
22:03:59.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:03:59.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.63.20:39889
22:03:59.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:03:59.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:03:59.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:03:59.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:03:59.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.63.62:41701,127.24.63.61:38085,127.24.63.60:44011
22:03:59.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:03:59.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34301/jwks.json with env {}
22:03:59.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.819386 24983 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.63.62" port: 41701 } has no permanent_uuid. Determining permanent_uuid...
22:03:59.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.840899 24983 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } has no permanent_uuid. Determining permanent_uuid...
22:03:59.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.855886 24983 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } has no permanent_uuid. Determining permanent_uuid...
22:03:59.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:59.858464 24934 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.63.60:44011: connect: Connection refused (error 111)
22:03:59.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:59.862927 24983 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.63.60:44011: Network error: Client connection negotiation failed: client connection to 127.24.63.60:44011: connect: Connection refused (error 111)
22:03:59.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.912459 24983 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } attempt: 1
22:03:59.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:59.916496 24983 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.63.60:44011: Network error: Client connection negotiation failed: client connection to 127.24.63.60:44011: connect: Connection refused (error 111)
22:03:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:03:59.985025 24983 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } attempt: 2
22:03:59.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:03:59.989012 24983 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.63.60:44011: Network error: Client connection negotiation failed: client connection to 127.24.63.60:44011: connect: Connection refused (error 111)
22:04:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:00.059643 24908 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } attempt: 7
22:04:00.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:00.071514 24908 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } has no permanent_uuid. Determining permanent_uuid...
22:04:00.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:00.077107 24908 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.63.60:44011: Network error: Client connection negotiation failed: client connection to 127.24.63.60:44011: connect: Connection refused (error 111)
22:04:00.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:00.080514 24983 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } attempt: 3
22:04:00.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:00.084697 24983 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.63.60:44011: Network error: Client connection negotiation failed: client connection to 127.24.63.60:44011: connect: Connection refused (error 111)
22:04:00.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:00.127593 24985 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:04:00.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:00.128152 24985 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:04:00.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:00.128638 24985 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:04:00.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:00.135527 24908 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } attempt: 1
22:04:00.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:00.139809 24908 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.63.60:44011: Network error: Client connection negotiation failed: client connection to 127.24.63.60:44011: connect: Connection refused (error 111)
22:04:00.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:00.161770 24985 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:04:00.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:00.162156 24985 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:04:00.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:00.162484 24985 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:04:00.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:00.162783 24985 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:04:00.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:00.163079 24985 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:04:00.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:00.163300 24985 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34301/jwks.json
22:04:00.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:00.199496 24985 master_runner.cc:386] Master server non-default flags:
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.63.20:39889
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-2/data
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-2/wal
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.63.62:41701,127.24.63.61:38085,127.24.63.60:44011
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.63.60:44011
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34301/jwks.json
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-2/data/info.pb
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.63.60
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-2/logs
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:04:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:04:00.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
22:04:00.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
22:04:00.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:04:00.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
22:04:00.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
22:04:00.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:04:00.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:00.201148 24985 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:04:00.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:00.202867 24985 file_cache.cc:492] Constructed file cache file cache with capacity 419430
22:04:00.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:00.214929 24997 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:04:00.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:00.221163 24908 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } attempt: 2
22:04:00.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:00.224937 24908 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.63.60:44011: Network error: Client connection negotiation failed: client connection to 127.24.63.60:44011: connect: Connection refused (error 111)
22:04:00.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:00.228150 24983 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } attempt: 4
22:04:00.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:00.232563 24983 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.63.60:44011: Network error: Client connection negotiation failed: client connection to 127.24.63.60:44011: connect: Connection refused (error 111)
22:04:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:00.310389 24908 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } attempt: 3
22:04:00.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:00.317437 24908 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.63.60:44011: Network error: Client connection negotiation failed: client connection to 127.24.63.60:44011: connect: Connection refused (error 111)
22:04:00.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:00.458169 24908 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } attempt: 4
22:04:00.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:00.465474 24908 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.63.60:44011: Network error: Client connection negotiation failed: client connection to 127.24.63.60:44011: connect: Connection refused (error 111)
22:04:00.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:00.532048 24983 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } attempt: 5
22:04:00.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:00.539017 24983 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.63.60:44011: Network error: Client connection negotiation failed: client connection to 127.24.63.60:44011: connect: Connection refused (error 111)
22:04:00.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:00.749186 24908 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } attempt: 5
22:04:00.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:00.756179 24908 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.63.60:44011: Network error: Client connection negotiation failed: client connection to 127.24.63.60:44011: connect: Connection refused (error 111)
22:04:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.086640 24983 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } attempt: 6
22:04:01.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:01.097441 24983 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.63.60:44011: Network error: Client connection negotiation failed: client connection to 127.24.63.60:44011: connect: Connection refused (error 111)
22:04:01.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.308800 24908 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } attempt: 6
22:04:01.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:01.321553 24908 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.63.60:44011: Network error: Client connection negotiation failed: client connection to 127.24.63.60:44011: connect: Connection refused (error 111)
22:04:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:00.215471 24998 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:04:01.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:01.453162 25000 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:04:01.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:01.456125 24999 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1235 milliseconds
22:04:01.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.456210 24985 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
22:04:01.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.457403 24985 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:04:01.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.460248 24985 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:04:01.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.461705 24985 hybrid_clock.cc:648] HybridClock initialized: now 1756850641461667 us; error 50 us; skew 500 ppm
22:04:01.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.462620 24985 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:04:01.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.469364 24985 webserver.cc:480] Webserver started at http://127.24.63.60:40461/ using document root <none> and password file <none>
22:04:01.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.470439 24985 fs_manager.cc:362] Metadata directory not provided
22:04:01.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.470692 24985 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:04:01.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.471186 24985 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
22:04:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.475749 24985 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-2/data/instance:
22:04:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae1367b7ac974aa28bec9be80c79d6c1"
22:04:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 22:04:01 on dist-test-slave-53g9"
22:04:01.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.476914 24985 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-2/wal/instance:
22:04:01.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae1367b7ac974aa28bec9be80c79d6c1"
22:04:01.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 22:04:01 on dist-test-slave-53g9"
22:04:01.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.484273 24985 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.003s
22:04:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.490387 25011 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:04:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.491451 24985 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
22:04:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.491876 24985 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-2/data,/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-2/wal
22:04:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae1367b7ac974aa28bec9be80c79d6c1"
22:04:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 22:04:01 on dist-test-slave-53g9"
22:04:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.492244 24985 fs_report.cc:389] FS layout report
22:04:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:04:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-2/wal
22:04:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-2/wal
22:04:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-2/data/data
22:04:01.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:04:01.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:04:01.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:04:01.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:04:01.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:04:01.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:04:01.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:04:01.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:04:01.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:04:01.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:04:01.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:04:01.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.548875 24985 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
22:04:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.550515 24985 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:04:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.551020 24985 kserver.cc:163] Server-wide thread pool size limit: 3276
22:04:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.622686 24985 rpc_server.cc:307] RPC server started. Bound to: 127.24.63.60:44011
22:04:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.622757 25063 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.63.60:44011 every 8 connection(s)
22:04:01.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.625504 24985 server_base.cc:1179] Dumped server information to /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-2/data/info.pb
22:04:01.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.629127 24828 external_mini_cluster.cc:1428] Started /tmp/dist-test-task3DGRBN/build/tsan/bin/kudu as pid 24985
22:04:01.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.629650 24828 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/master-2/wal/instance
22:04:01.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.631930 25064 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:04:01.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.648483 25064 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.63.62" port: 41701 } has no permanent_uuid. Determining permanent_uuid...
22:04:01.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.672521 25064 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } has no permanent_uuid. Determining permanent_uuid...
22:04:01.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.684289 25064 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } has no permanent_uuid. Determining permanent_uuid...
22:04:01.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.702152 25064 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1: Bootstrap starting.
22:04:01.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.708650 25064 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1: Neither blocks nor log segments found. Creating new log.
22:04:01.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.710423 25064 log.cc:826] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1: Log is configured to *not* fsync() on all Append() calls
22:04:01.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.714656 25064 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1: No bootstrap required, opened a new log
22:04:01.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.733139 25064 raft_consensus.cc:357] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1e623a08940348cab7185f81195de1d6" member_type: VOTER last_known_addr { host: "127.24.63.62" port: 41701 } } peers { permanent_uuid: "a0a91eb6a56442589425efb2c6316870" member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } } peers { permanent_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } }
22:04:01.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.733793 25064 raft_consensus.cc:383] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
22:04:01.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.734005 25064 raft_consensus.cc:738] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ae1367b7ac974aa28bec9be80c79d6c1, State: Initialized, Role: FOLLOWER
22:04:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.734814 25064 consensus_queue.cc:260] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1e623a08940348cab7185f81195de1d6" member_type: VOTER last_known_addr { host: "127.24.63.62" port: 41701 } } peers { permanent_uuid: "a0a91eb6a56442589425efb2c6316870" member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } } peers { permanent_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } }
22:04:01.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.737299 25073 sys_catalog.cc:455] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1e623a08940348cab7185f81195de1d6" member_type: VOTER last_known_addr { host: "127.24.63.62" port: 41701 } } peers { permanent_uuid: "a0a91eb6a56442589425efb2c6316870" member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } } peers { permanent_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } } }
22:04:01.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.738332 25073 sys_catalog.cc:458] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [sys.catalog]: This master's current role is: FOLLOWER
22:04:01.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.738926 25064 sys_catalog.cc:564] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [sys.catalog]: configured and running, proceeding with master startup.
22:04:01.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:01.762662 25084 catalog_manager.cc:1560] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:04:01.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:01.763100 25084 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:04:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.759629 24985 master_runner.cc:186] Error getting master registration for 127.24.63.62:41701: OK, instance_id { permanent_uuid: "1e623a08940348cab7185f81195de1d6" instance_seqno: 1756850637659488 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
22:04:01.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:01.793685 24985 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:04:01.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:01.806308 25017 tablet.cc:2378] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1: Can't schedule compaction. Clean time has not been advanced past its initial value.
22:04:02.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.158103 24983 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } attempt: 7
22:04:02.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.175706 24983 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870: Bootstrap starting.
22:04:02.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.184767 24983 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870: Neither blocks nor log segments found. Creating new log.
22:04:02.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.187053 24983 log.cc:826] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870: Log is configured to *not* fsync() on all Append() calls
22:04:02.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.192229 24983 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870: No bootstrap required, opened a new log
22:04:02.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.205720 25073 raft_consensus.cc:491] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
22:04:02.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.206271 25073 raft_consensus.cc:513] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1e623a08940348cab7185f81195de1d6" member_type: VOTER last_known_addr { host: "127.24.63.62" port: 41701 } } peers { permanent_uuid: "a0a91eb6a56442589425efb2c6316870" member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } } peers { permanent_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } }
22:04:02.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.208519 25073 leader_election.cc:290] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1e623a08940348cab7185f81195de1d6 (127.24.63.62:41701), a0a91eb6a56442589425efb2c6316870 (127.24.63.61:38085)
22:04:02.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.221431 24983 raft_consensus.cc:357] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1e623a08940348cab7185f81195de1d6" member_type: VOTER last_known_addr { host: "127.24.63.62" port: 41701 } } peers { permanent_uuid: "a0a91eb6a56442589425efb2c6316870" member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } } peers { permanent_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } }
22:04:02.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.222333 24983 raft_consensus.cc:383] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
22:04:02.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.222635 24983 raft_consensus.cc:738] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a0a91eb6a56442589425efb2c6316870, State: Initialized, Role: FOLLOWER
22:04:02.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.223556 24983 consensus_queue.cc:260] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1e623a08940348cab7185f81195de1d6" member_type: VOTER last_known_addr { host: "127.24.63.62" port: 41701 } } peers { permanent_uuid: "a0a91eb6a56442589425efb2c6316870" member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } } peers { permanent_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } }
22:04:02.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.226382 25088 sys_catalog.cc:455] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1e623a08940348cab7185f81195de1d6" member_type: VOTER last_known_addr { host: "127.24.63.62" port: 41701 } } peers { permanent_uuid: "a0a91eb6a56442589425efb2c6316870" member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } } peers { permanent_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } } }
22:04:02.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.227406 25088 sys_catalog.cc:458] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870 [sys.catalog]: This master's current role is: FOLLOWER
22:04:02.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.228435 24983 sys_catalog.cc:564] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870 [sys.catalog]: configured and running, proceeding with master startup.
22:04:02.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.228679 24958 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0a91eb6a56442589425efb2c6316870" is_pre_election: true
22:04:02.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.229583 24958 raft_consensus.cc:2466] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ae1367b7ac974aa28bec9be80c79d6c1 in term 0.
22:04:02.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.231060 25013 leader_election.cc:304] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a0a91eb6a56442589425efb2c6316870, ae1367b7ac974aa28bec9be80c79d6c1; no voters:
22:04:02.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.232141 25073 raft_consensus.cc:2802] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [term 0 FOLLOWER]: Leader pre-election won for term 1
22:04:02.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.232578 25073 raft_consensus.cc:491] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
22:04:02.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.232975 25073 raft_consensus.cc:3058] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [term 0 FOLLOWER]: Advancing to term 1
22:04:02.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:02.233507 24937 tablet.cc:2378] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870: Can't schedule compaction. Clean time has not been advanced past its initial value.
22:04:02.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.240188 25073 raft_consensus.cc:513] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1e623a08940348cab7185f81195de1d6" member_type: VOTER last_known_addr { host: "127.24.63.62" port: 41701 } } peers { permanent_uuid: "a0a91eb6a56442589425efb2c6316870" member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } } peers { permanent_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } }
22:04:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.243557 25073 leader_election.cc:290] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [CANDIDATE]: Term 1 election: Requested vote from peers 1e623a08940348cab7185f81195de1d6 (127.24.63.62:41701), a0a91eb6a56442589425efb2c6316870 (127.24.63.61:38085)
22:04:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.243705 24958 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0a91eb6a56442589425efb2c6316870"
22:04:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.244220 24958 raft_consensus.cc:3058] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870 [term 0 FOLLOWER]: Advancing to term 1
22:04:02.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.256924 24958 raft_consensus.cc:2466] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ae1367b7ac974aa28bec9be80c79d6c1 in term 1.
22:04:02.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.258193 25013 leader_election.cc:304] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a0a91eb6a56442589425efb2c6316870, ae1367b7ac974aa28bec9be80c79d6c1; no voters:
22:04:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.258880 25073 raft_consensus.cc:2802] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [term 1 FOLLOWER]: Leader election won for term 1
22:04:02.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.261276 25073 raft_consensus.cc:695] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [term 1 LEADER]: Becoming Leader. State: Replica: ae1367b7ac974aa28bec9be80c79d6c1, State: Running, Role: LEADER
22:04:02.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.262374 25073 consensus_queue.cc:237] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1e623a08940348cab7185f81195de1d6" member_type: VOTER last_known_addr { host: "127.24.63.62" port: 41701 } } peers { permanent_uuid: "a0a91eb6a56442589425efb2c6316870" member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } } peers { permanent_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } }
22:04:02.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.301558 25095 sys_catalog.cc:455] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ae1367b7ac974aa28bec9be80c79d6c1. Latest consensus state: current_term: 1 leader_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1e623a08940348cab7185f81195de1d6" member_type: VOTER last_known_addr { host: "127.24.63.62" port: 41701 } } peers { permanent_uuid: "a0a91eb6a56442589425efb2c6316870" member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } } peers { permanent_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } } }
22:04:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.307592 25095 sys_catalog.cc:458] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [sys.catalog]: This master's current role is: LEADER
22:04:02.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.312088 25103 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
22:04:02.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:02.320436 25101 catalog_manager.cc:1560] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:04:02.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:02.320871 25101 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:04:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.331372 25103 catalog_manager.cc:1486] Initializing Kudu cluster ID...
22:04:02.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.284795 24910 master_runner.cc:186] Error getting master registration for 127.24.63.62:41701: OK, instance_id { permanent_uuid: "1e623a08940348cab7185f81195de1d6" instance_seqno: 1756850637659488 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
22:04:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.343210 24910 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:04:02.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.351454 24958 raft_consensus.cc:1273] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870 [term 1 FOLLOWER]: Refusing update from remote peer ae1367b7ac974aa28bec9be80c79d6c1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
22:04:02.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.353588 25095 consensus_queue.cc:1035] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a0a91eb6a56442589425efb2c6316870" member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
22:04:02.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.354964 24908 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } attempt: 7
22:04:02.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.397253 25088 sys_catalog.cc:455] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ae1367b7ac974aa28bec9be80c79d6c1. Latest consensus state: current_term: 1 leader_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1e623a08940348cab7185f81195de1d6" member_type: VOTER last_known_addr { host: "127.24.63.62" port: 41701 } } peers { permanent_uuid: "a0a91eb6a56442589425efb2c6316870" member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } } peers { permanent_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } } }
22:04:02.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.399972 25088 sys_catalog.cc:458] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870 [sys.catalog]: This master's current role is: FOLLOWER
22:04:02.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.414937 25095 sys_catalog.cc:455] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1e623a08940348cab7185f81195de1d6" member_type: VOTER last_known_addr { host: "127.24.63.62" port: 41701 } } peers { permanent_uuid: "a0a91eb6a56442589425efb2c6316870" member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } } peers { permanent_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } } }
22:04:02.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.415741 25095 sys_catalog.cc:458] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [sys.catalog]: This master's current role is: LEADER
22:04:02.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.417177 25103 catalog_manager.cc:1349] Generated new cluster ID: bd6ee55af5434fa2bcdc6de55bfaf719
22:04:02.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.417493 25103 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
22:04:02.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.425383 24908 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1e623a08940348cab7185f81195de1d6: Bootstrap starting.
22:04:02.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.416541 25106 mvcc.cc:204] Tried to move back new op lower bound from 7196060231053799424 to 7196060230717022208. Current Snapshot: MvccSnapshot[applied={T|T < 7196060231053799424}]
22:04:02.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.431836 25088 sys_catalog.cc:455] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1e623a08940348cab7185f81195de1d6" member_type: VOTER last_known_addr { host: "127.24.63.62" port: 41701 } } peers { permanent_uuid: "a0a91eb6a56442589425efb2c6316870" member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } } peers { permanent_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } } }
22:04:02.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.437172 25088 sys_catalog.cc:458] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870 [sys.catalog]: This master's current role is: FOLLOWER
22:04:02.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.461680 24908 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1e623a08940348cab7185f81195de1d6: Neither blocks nor log segments found. Creating new log.
22:04:02.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.464591 24908 log.cc:826] T 00000000000000000000000000000000 P 1e623a08940348cab7185f81195de1d6: Log is configured to *not* fsync() on all Append() calls
22:04:02.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.480276 24908 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1e623a08940348cab7185f81195de1d6: No bootstrap required, opened a new log
22:04:02.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.482420 25103 catalog_manager.cc:1372] Generated new certificate authority record
22:04:02.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.484481 25103 catalog_manager.cc:1506] Loading token signing keys...
22:04:02.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.511682 24908 raft_consensus.cc:357] T 00000000000000000000000000000000 P 1e623a08940348cab7185f81195de1d6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1e623a08940348cab7185f81195de1d6" member_type: VOTER last_known_addr { host: "127.24.63.62" port: 41701 } } peers { permanent_uuid: "a0a91eb6a56442589425efb2c6316870" member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } } peers { permanent_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } }
22:04:02.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.512806 24908 raft_consensus.cc:383] T 00000000000000000000000000000000 P 1e623a08940348cab7185f81195de1d6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
22:04:02.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.513226 24908 raft_consensus.cc:738] T 00000000000000000000000000000000 P 1e623a08940348cab7185f81195de1d6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1e623a08940348cab7185f81195de1d6, State: Initialized, Role: FOLLOWER
22:04:02.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.514410 24908 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1e623a08940348cab7185f81195de1d6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1e623a08940348cab7185f81195de1d6" member_type: VOTER last_known_addr { host: "127.24.63.62" port: 41701 } } peers { permanent_uuid: "a0a91eb6a56442589425efb2c6316870" member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } } peers { permanent_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } }
22:04:02.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.526108 24908 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1e623a08940348cab7185f81195de1d6 [sys.catalog]: configured and running, proceeding with master startup.
22:04:02.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.527280 25111 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1e623a08940348cab7185f81195de1d6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1e623a08940348cab7185f81195de1d6" member_type: VOTER last_known_addr { host: "127.24.63.62" port: 41701 } } peers { permanent_uuid: "a0a91eb6a56442589425efb2c6316870" member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } } peers { permanent_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } } }
22:04:02.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.527478 24881 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 1e623a08940348cab7185f81195de1d6 [term 0 FOLLOWER]: Advancing to term 1
22:04:02.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.528136 24883 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1e623a08940348cab7185f81195de1d6" is_pre_election: true
22:04:02.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.526291 24882 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1e623a08940348cab7185f81195de1d6"
22:04:02.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.538878 24881 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 1e623a08940348cab7185f81195de1d6 [term 1 FOLLOWER]: Refusing update from remote peer ae1367b7ac974aa28bec9be80c79d6c1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
22:04:02.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.540400 25111 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1e623a08940348cab7185f81195de1d6 [sys.catalog]: This master's current role is: FOLLOWER
22:04:02.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.540635 25073 consensus_queue.cc:1035] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1e623a08940348cab7185f81195de1d6" member_type: VOTER last_known_addr { host: "127.24.63.62" port: 41701 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
22:04:02.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.593576 25103 catalog_manager.cc:5955] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1: Generated new TSK 0
22:04:02.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.594822 25103 catalog_manager.cc:1516] Initializing in-progress tserver states...
22:04:02.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.624850 25120 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1e623a08940348cab7185f81195de1d6 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1e623a08940348cab7185f81195de1d6" member_type: VOTER last_known_addr { host: "127.24.63.62" port: 41701 } } peers { permanent_uuid: "a0a91eb6a56442589425efb2c6316870" member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } } peers { permanent_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } } }
22:04:02.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.626535 25111 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1e623a08940348cab7185f81195de1d6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ae1367b7ac974aa28bec9be80c79d6c1. Latest consensus state: current_term: 1 leader_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1e623a08940348cab7185f81195de1d6" member_type: VOTER last_known_addr { host: "127.24.63.62" port: 41701 } } peers { permanent_uuid: "a0a91eb6a56442589425efb2c6316870" member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } } peers { permanent_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } } }
22:04:02.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.627429 25111 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1e623a08940348cab7185f81195de1d6 [sys.catalog]: This master's current role is: FOLLOWER
22:04:02.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.628103 25120 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1e623a08940348cab7185f81195de1d6 [sys.catalog]: This master's current role is: FOLLOWER
22:04:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.643577 24828 external_mini_cluster.cc:1366] Running /tmp/dist-test-task3DGRBN/build/tsan/bin/kudu
22:04:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task3DGRBN/build/tsan/bin/kudu
22:04:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/ts-0/wal
22:04:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/ts-0/data
22:04:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:04:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:04:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:04:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:04:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:04:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:04:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/ts-0/logs
22:04:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/ts-0/data/info.pb
22:04:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:04:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:04:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:04:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:04:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:04:02.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:04:02.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:04:02.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:04:02.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
22:04:02.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:04:02.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.63.1:0
22:04:02.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.63.1
22:04:02.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.63.1
22:04:02.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:04:02.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.63.62:41701,127.24.63.61:38085,127.24.63.60:44011
22:04:02.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.63.20:39889
22:04:02.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:04:02.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:04:02.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
22:04:02.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.639125 25095 sys_catalog.cc:455] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1e623a08940348cab7185f81195de1d6" member_type: VOTER last_known_addr { host: "127.24.63.62" port: 41701 } } peers { permanent_uuid: "a0a91eb6a56442589425efb2c6316870" member_type: VOTER last_known_addr { host: "127.24.63.61" port: 38085 } } peers { permanent_uuid: "ae1367b7ac974aa28bec9be80c79d6c1" member_type: VOTER last_known_addr { host: "127.24.63.60" port: 44011 } } }
22:04:02.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.647265 25095 sys_catalog.cc:458] T 00000000000000000000000000000000 P ae1367b7ac974aa28bec9be80c79d6c1 [sys.catalog]: This master's current role is: LEADER
22:04:02.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:02.665262 25129 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
22:04:03.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:03.063607 25128 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:04:03.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:03.064198 25128 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:04:03.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:03.064774 25128 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:04:03.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:03.099115 25128 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:04:03.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:03.100284 25128 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.63.1
22:04:03.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:03.138512 25128 tablet_server_runner.cc:78] Tablet server non-default flags:
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.63.20:39889
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/ts-0/data
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/ts-0/wal
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.63.1:0
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/ts-0/data/info.pb
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.63.1
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.63.62:41701,127.24.63.61:38085,127.24.63.60:44011
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.63.1
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/ts-0/logs
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6ab3f249351a4ee5d73eb8bda3585c4dcd85d070
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 21:30:03 UTC on 5fd53c4cbb9d
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7869
22:04:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:04:03.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:03.140199 25128 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:04:03.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:03.141927 25128 file_cache.cc:492] Constructed file cache file cache with capacity 419430
22:04:03.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:03.158413 25135 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:04:03.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:03.330333 25101 catalog_manager.cc:1261] Loaded cluster ID: bd6ee55af5434fa2bcdc6de55bfaf719
22:04:03.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:03.331123 25101 catalog_manager.cc:1554] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870: loading cluster ID for follower catalog manager: success
22:04:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:03.344164 25101 catalog_manager.cc:1576] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870: acquiring CA information for follower catalog manager: success
22:04:03.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:03.358820 25101 catalog_manager.cc:1604] T 00000000000000000000000000000000 P a0a91eb6a56442589425efb2c6316870: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
22:04:03.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:03.682967 25129 catalog_manager.cc:1261] Loaded cluster ID: bd6ee55af5434fa2bcdc6de55bfaf719
22:04:03.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:03.683495 25129 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 1e623a08940348cab7185f81195de1d6: loading cluster ID for follower catalog manager: success
22:04:03.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:03.690963 25129 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 1e623a08940348cab7185f81195de1d6: acquiring CA information for follower catalog manager: success
22:04:03.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:03.707374 25129 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 1e623a08940348cab7185f81195de1d6: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
22:04:04.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) LLVMSymbolizer: error reading file: No such file or directory
22:04:04.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) LLVMSymbolizer: error reading file: No such file or directory
22:04:04.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) LLVMSymbolizer: error reading file: No such file or directory
22:04:04.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:03.171272 25128 server_base.cc:1047] running on GCE node
22:04:04.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:03.172375 25138 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:04:04.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:03.170408 25136 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:04:04.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:04.543891 25128 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:04:04.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:04.555583 25128 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:04:04.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:04.563349 25128 hybrid_clock.cc:648] HybridClock initialized: now 1756850644563274 us; error 68 us; skew 500 ppm
22:04:04.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:04.564560 25128 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:04:04.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:04.588336 25128 webserver.cc:480] Webserver started at http://127.24.63.1:39123/ using document root <none> and password file <none>
22:04:04.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:04.590021 25128 fs_manager.cc:362] Metadata directory not provided
22:04:04.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:04.590487 25128 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:04:04.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:04.590915 25128 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
22:04:04.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:04.591775 25128 fs_manager.cc:362] Metadata directory not provided
22:04:04.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:04.592085 25128 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:04:04.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:04.593024 25128 ts_tablet_manager.cc:1500] Shutting down tablet manager...
22:04:04.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) LLVMSymbolizer: error reading file: No such file or directory
22:04:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
22:04:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=25128)
22:04:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 4 at 0x7b6800001800 by main thread (mutexes: write M3753):
22:04:04.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1c0ed4)
22:04:04.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x18f172)
22:04:04.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x143e92)
22:04:04.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libserver_process.so+0x143c26)
22:04:04.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmaster.so+0x401a1f)
22:04:04.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libtserver.so+0x255e62)
22:04:04.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libtserver.so+0x263e8a)
22:04:04.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 kudu::tools::(anonymous namespace)::TServerRun(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action_tserver.cc:114:10 (kudu+0x629109)
22:04:04.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (kudu+0x529e81)
22:04:04.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (kudu+0x529dec)
22:04:04.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (kudu+0x529d5c)
22:04:04.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (kudu+0x5289c5)
22:04:04.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x13fc1f)
22:04:04.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x132481)
22:04:04.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action.cc:245:10 (libkudu_tools_util.so+0x130212)
22:04:04.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:142:15 (kudu+0x645090)
22:04:04.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:214:16 (kudu+0x6459c7)
22:04:04.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:309:10 (kudu+0x645f0e)
22:04:04.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:04:04.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous read of size 4 at 0x7b6800001800 by thread T12:
22:04:04.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1c2a2f)
22:04:04.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:04:04.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Location is heap block of size 1376 at 0x7b6800001800 allocated by main thread:
22:04:04.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 calloc /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:669 (kudu+0x498efc)
22:04:04.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c1404)
22:04:04.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x1900f0)
22:04:04.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libserver_process.so+0x144a8e)
22:04:04.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libkserver.so+0x12ab1)
22:04:04.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libtserver.so+0x2564d2)
22:04:04.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libtserver.so+0x263e01)
22:04:04.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 kudu::tools::(anonymous namespace)::TServerRun(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action_tserver.cc:114:10 (kudu+0x629109)
22:04:04.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (kudu+0x529e81)
22:04:04.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (kudu+0x529dec)
22:04:04.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (kudu+0x529d5c)
22:04:04.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (kudu+0x5289c5)
22:04:04.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x13fc1f)
22:04:04.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x132481)
22:04:04.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action.cc:245:10 (libkudu_tools_util.so+0x130212)
22:04:04.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:142:15 (kudu+0x645090)
22:04:04.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:214:16 (kudu+0x6459c7)
22:04:04.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:309:10 (kudu+0x645f0e)
22:04:04.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:04:04.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mutex M3753 (0x7b6800001980) created at:
22:04:04.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 pthread_mutex_init /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1224 (kudu+0x46cd02)
22:04:04.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c24ef)
22:04:04.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x1900f0)
22:04:04.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libserver_process.so+0x144a8e)
22:04:04.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libkserver.so+0x12ab1)
22:04:04.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libtserver.so+0x2564d2)
22:04:04.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libtserver.so+0x263e01)
22:04:04.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 kudu::tools::(anonymous namespace)::TServerRun(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action_tserver.cc:114:10 (kudu+0x629109)
22:04:04.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (kudu+0x529e81)
22:04:04.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (kudu+0x529dec)
22:04:04.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (kudu+0x529d5c)
22:04:04.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (kudu+0x5289c5)
22:04:04.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x13fc1f)
22:04:04.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x132481)
22:04:04.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action.cc:245:10 (libkudu_tools_util.so+0x130212)
22:04:04.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:142:15 (kudu+0x645090)
22:04:04.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:214:16 (kudu+0x6459c7)
22:04:04.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:309:10 (kudu+0x645f0e)
22:04:04.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:04:04.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T12 'sq_acceptor' (tid=25148, running) created by main thread at:
22:04:04.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x46c795)
22:04:04.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c278d)
22:04:04.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x1900f0)
22:04:04.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libserver_process.so+0x144a8e)
22:04:04.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libkserver.so+0x12ab1)
22:04:04.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libtserver.so+0x2564d2)
22:04:04.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libtserver.so+0x263e01)
22:04:04.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 kudu::tools::(anonymous namespace)::TServerRun(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action_tserver.cc:114:10 (kudu+0x629109)
22:04:04.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (kudu+0x529e81)
22:04:04.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (kudu+0x529dec)
22:04:04.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (kudu+0x529d5c)
22:04:04.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (kudu+0x5289c5)
22:04:04.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x13fc1f)
22:04:04.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x132481)
22:04:04.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action.cc:245:10 (libkudu_tools_util.so+0x130212)
22:04:04.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:142:15 (kudu+0x645090)
22:04:04.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:214:16 (kudu+0x6459c7)
22:04:04.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:309:10 (kudu+0x645f0e)
22:04:04.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:04:04.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-task3DGRBN/build/tsan/lib/libserver_process.so+0x1c0ed4)
22:04:04.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
22:04:04.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
22:04:04.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=25128)
22:04:04.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 8 at 0x7b0c00031da0 by thread T12:
22:04:04.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 free /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:708 (kudu+0x49679f)
22:04:04.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c2ac0)
22:04:04.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:04:04.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous read of size 4 at 0x7b0c00031da0 by main thread:
22:04:04.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1bbcbd)
22:04:04.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x190ca7)
22:04:04.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x1901d8)
22:04:04.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libserver_process.so+0x144a8e)
22:04:04.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libkserver.so+0x12ab1)
22:04:04.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libtserver.so+0x2564d2)
22:04:04.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libtserver.so+0x263e01)
22:04:04.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 kudu::tools::(anonymous namespace)::TServerRun(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action_tserver.cc:114:10 (kudu+0x629109)
22:04:04.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (kudu+0x529e81)
22:04:04.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (kudu+0x529dec)
22:04:04.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (kudu+0x529d5c)
22:04:04.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (kudu+0x5289c5)
22:04:04.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x13fc1f)
22:04:04.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x132481)
22:04:04.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action.cc:245:10 (libkudu_tools_util.so+0x130212)
22:04:04.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:142:15 (kudu+0x645090)
22:04:04.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:214:16 (kudu+0x6459c7)
22:04:04.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:309:10 (kudu+0x645f0e)
22:04:04.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:04:04.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T12 'sq_acceptor' (tid=25148, running) created by main thread at:
22:04:04.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 pthread_create /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:966 (kudu+0x46c795)
22:04:04.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c278d)
22:04:04.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x1900f0)
22:04:04.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libserver_process.so+0x144a8e)
22:04:04.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libkserver.so+0x12ab1)
22:04:04.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libtserver.so+0x2564d2)
22:04:04.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libtserver.so+0x263e01)
22:04:04.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 kudu::tools::(anonymous namespace)::TServerRun(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action_tserver.cc:114:10 (kudu+0x629109)
22:04:04.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 decltype(std::__1::forward<kudu::Status (*&)(kudu::tools::RunnerContext const&)>(fp)(std::__1::forward<kudu::tools::RunnerContext const&>(fp0))) std::__1::__invoke<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/type_traits:3905:1 (kudu+0x529e81)
22:04:04.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/__functional_base:317:16 (kudu+0x529dec)
22:04:04.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 std::__1::__function::__alloc_func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (kudu+0x529d5c)
22:04:04.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (kudu+0x5289c5)
22:04:04.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 std::__1::__function::__value_func<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libkudu_tools_util.so+0x13fc1f)
22:04:04.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libkudu_tools_util.so+0x132481)
22:04:04.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_action.cc:245:10 (libkudu_tools_util.so+0x130212)
22:04:04.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:142:15 (kudu+0x645090)
22:04:04.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 kudu::tools::RunTool(int, char**, bool) /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:214:16 (kudu+0x6459c7)
22:04:04.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 main /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/tools/tool_main.cc:309:10 (kudu+0x645f0e)
22:04:04.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:04:04.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-task3DGRBN/build/tsan/lib/libserver_process.so+0x1c2ac0)
22:04:04.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
22:04:04.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:04.641909 25128 builtin_ntp.cc:688] coult not shutdown socket: Network error: shutdown error: Transport endpoint is not connected (error 107)
22:04:04.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Could not create new FS layout: Write-ahead log directory /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/ts-0/wal failed to canonicalize: Unable to canonicalize /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898/ts-0: No such file or directory (error 2)
22:04:05.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ThreadSanitizer: reported 2 warnings
22:04:05.689 [DEBUG - main] (MiniKuduCluster.java:195) Response: error {
code: RUNTIME_ERROR
message: "failed to start tablet server 1: /tmp/dist-test-task3DGRBN/build/tsan/bin/kudu: process exited with non-zero status 66"
}
22:04:05.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:05.690281 24828 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task3DGRBN/build/tsan/bin/kudu with pid 24840
22:04:05.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:05.719236 24828 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task3DGRBN/build/tsan/bin/kudu with pid 24910
22:04:05.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 22:04:05.750723 24828 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task3DGRBN/build/tsan/bin/kudu with pid 24985
22:04:05.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:05.779167 24828 env_posix.cc:2133] Unable to access file /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898 during walk: No such file or directory
22:04:05.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:05.779517 24828 env_posix.cc:2135] Unable to access file /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898 during walk: No such file or directory
22:04:05.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 22:04:05.779876 24828 tool_action_test.cc:443] Could not delete cluster root: IO error: /tmp/dist-test-task3DGRBN/test-tmp/mini-kudu-cluster6062635096149791898: One or more errors occurred
22:04:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==24828==WARNING: invalid path to external symbolizer!
22:04:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==24828==WARNING: Failed to use and restart external symbolizer!
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=24828)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 4 at 0x7b6800001e00 by main thread (mutexes: write M3457):
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1c0ed4)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x18f172)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libmini_oidc.so+0x21aa2)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x219ea)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0xb3e5e)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libmini_cluster.so+0x993cd)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libmini_cluster.so+0x9836b)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libmini_cluster.so+0x846c9)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libmini_cluster.so+0x84829)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x6585e7)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x654a5d)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x65412b)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x651350)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x529e81)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x529dec)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x529d5c)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x5289c5)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libkudu_tools_util.so+0x13fc1f)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libkudu_tools_util.so+0x132481)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (libkudu_tools_util.so+0x130212)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (kudu+0x645090)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #21 <null> <null> (kudu+0x6459c7)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #22 <null> <null> (kudu+0x645f0e)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #23 <null> <null> (libc.so.6+0x21bf6)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous read of size 4 at 0x7b6800001e00 by thread T5:
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1c2a2f)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (kudu+0x468cdf)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) As if synchronized via sleep:
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x48dcc2)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libgutil.so+0xb5f0a)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libkudu_util.so+0x3d7f96)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_cluster.so+0x90bce)
22:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x94207)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libmini_cluster.so+0x88a6f)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libmini_cluster.so+0x86caa)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x652939)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x651234)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x529e81)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x529dec)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x529d5c)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x5289c5)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x13fc1f)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x132481)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (libkudu_tools_util.so+0x130212)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x645090)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x6459c7)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (kudu+0x645f0e)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (libc.so.6+0x21bf6)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Location is heap block of size 1376 at 0x7b6800001e00 allocated by main thread:
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x498efc)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c1404)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x1900f0)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x22185)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x857e4)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x652939)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x651234)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x529e81)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x529dec)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x529d5c)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x5289c5)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13fc1f)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x132481)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x130212)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x645090)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6459c7)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x645f0e)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mutex M3457 (0x7b6800001f80) created at:
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x46cd02)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c24ef)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x1900f0)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x22185)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x857e4)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x652939)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x651234)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x529e81)
22:04:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x529dec)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x529d5c)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x5289c5)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13fc1f)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x132481)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x130212)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x645090)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6459c7)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x645f0e)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T5 'sq_acceptor' (tid=24835, running) created by main thread at:
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x46c795)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c278d)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x1900f0)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x22185)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x857e4)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x652939)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x651234)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x529e81)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x529dec)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x529d5c)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x5289c5)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13fc1f)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x132481)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x130212)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x645090)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6459c7)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x645f0e)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-task3DGRBN/build/tsan/lib/libserver_process.so+0x1c0ed4)
22:04:05.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=24828)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Read of size 4 at 0x7b6800001800 by thread T4:
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1c2e97)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (kudu+0x468cdf)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous write of size 4 at 0x7b6800001800 by main thread (mutexes: write M3454):
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1c0ed4)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x18f172)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libmini_oidc.so+0x21ac5)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x219ea)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0xb3e5e)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libmini_cluster.so+0x993cd)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libmini_cluster.so+0x9836b)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libmini_cluster.so+0x846c9)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libmini_cluster.so+0x84829)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x6585e7)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x654a5d)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x65412b)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x651350)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x529e81)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x529dec)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x529d5c)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x5289c5)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libkudu_tools_util.so+0x13fc1f)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libkudu_tools_util.so+0x132481)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (libkudu_tools_util.so+0x130212)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (kudu+0x645090)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #21 <null> <null> (kudu+0x6459c7)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #22 <null> <null> (kudu+0x645f0e)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #23 <null> <null> (libc.so.6+0x21bf6)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Location is heap block of size 1376 at 0x7b6800001800 allocated by main thread:
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x498efc)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c1404)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x1900f0)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x21dd0)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x857e4)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x652939)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x651234)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x529e81)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x529dec)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x529d5c)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x5289c5)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13fc1f)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x132481)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x130212)
22:04:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x645090)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6459c7)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x645f0e)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mutex M3454 (0x7b6800001980) created at:
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x46cd02)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c24ef)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x1900f0)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x21dd0)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x857e4)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x652939)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x651234)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x529e81)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x529dec)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x529d5c)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x5289c5)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13fc1f)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x132481)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x130212)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x645090)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6459c7)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x645f0e)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T4 'sq_acceptor' (tid=24834, running) created by main thread at:
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x46c795)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c278d)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x1900f0)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x21dd0)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x857e4)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x652939)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x651234)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x529e81)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x529dec)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x529d5c)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x5289c5)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13fc1f)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x132481)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x130212)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x645090)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6459c7)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x645f0e)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-task3DGRBN/build/tsan/lib/libserver_process.so+0x1c2e97)
22:04:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
22:04:05.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T22:04:05Z chronyd exiting
22:04:06.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ThreadSanitizer: reported 2 warnings
22:04:06.859 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduClient.testCreateTableWithConcurrentInsert: failed attempt 1
java.io.IOException: failed to start tablet server 1: /tmp/dist-test-task3DGRBN/build/tsan/bin/kudu: process exited with non-zero status 66
at org.apache.kudu.test.cluster.MiniKuduCluster.sendRequestToCluster(MiniKuduCluster.java:199) ~[kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster.start(MiniKuduCluster.java:266) ~[kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster.access$300(MiniKuduCluster.java:85) ~[kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster$MiniKuduClusterBuilder.build(MiniKuduCluster.java:897) ~[kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:156) ~[kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.io.IOException: failed to start tablet server 1: /tmp/dist-test-task3DGRBN/build/tsan/bin/kudu: process exited with non-zero status 66
at org.apache.kudu.test.cluster.MiniKuduCluster.sendRequestToCluster(MiniKuduCluster.java:199)
at org.apache.kudu.test.cluster.MiniKuduCluster.start(MiniKuduCluster.java:266)
at org.apache.kudu.test.cluster.MiniKuduCluster.access$300(MiniKuduCluster.java:85)
at org.apache.kudu.test.cluster.MiniKuduCluster$MiniKuduClusterBuilder.build(MiniKuduCluster.java:897)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:156)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)