14:22:24.371 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
14:22:24.371 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task61f5S1/test-tmp
14:22:24.371 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:22:24.371 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task61f5S1/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:22:24.373 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824"
extra_master_flags: "--table_locations_ttl_ms=500"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
14:22:24.697 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:22:24.697 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:22:24.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:24.700589 28048 mini_oidc.cc:150] Starting JWKS server
14:22:24.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:24.703697 28048 webserver.cc:480] Webserver started at http://127.0.0.1:37497/ using document root <none> and password file <none>
14:22:24.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:24.704118 28048 mini_oidc.cc:186] Starting OIDC Discovery server
14:22:24.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:24.705840 28048 webserver.cc:480] Webserver started at http://127.0.0.1:35761/ using document root <none> and password file <none>
14:22:24.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-04T14:22:24Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:22:24.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-04T14:22:24Z Disabled control of system clock
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:24.768803 28048 external_mini_cluster.cc:1366] Running /tmp/dist-test-task61f5S1/build/tsan/bin/kudu
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task61f5S1/build/tsan/bin/kudu
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-0/wal
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-0/data
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-0/logs
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-0/data/info.pb
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.100.62:38363
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.100.62
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.100.20:43003
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.100.62:38363,127.27.100.61:36055,127.27.100.60:40945
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37497/jwks.json
14:22:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --table_locations_ttl_ms=500 with env {}
14:22:25.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:25.069674 28060 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:22:25.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:25.070250 28060 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:22:25.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:25.070734 28060 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:22:25.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:25.102425 28060 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:22:25.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:25.102710 28060 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:22:25.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:25.102905 28060 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:22:25.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:25.103107 28060 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:22:25.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:25.103294 28060 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:22:25.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:25.103436 28060 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37497/jwks.json
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:25.139779 28060 master_runner.cc:386] Master server non-default flags:
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.100.20:43003
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-0/data
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-0/wal
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --table_locations_ttl_ms=500
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.100.62:38363,127.27.100.61:36055,127.27.100.60:40945
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.100.62:38363
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37497/jwks.json
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-0/data/info.pb
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.100.62
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-0/logs
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:22:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:22:25.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:22:25.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:22:25.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 13:43:25 UTC on 5fd53c4cbb9d
14:22:25.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7897
14:22:25.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:22:25.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:25.141181 28060 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:22:25.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:25.142755 28060 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:22:25.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:25.153537 28066 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:26.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:25.154075 28067 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:26.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:26.348591 28069 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:26.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:26.350579 28068 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1190 milliseconds
14:22:26.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:26.351269 28060 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.197s user 0.414s sys 0.773s
14:22:26.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:26.351547 28060 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.198s user 0.414s sys 0.773s
14:22:26.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.351801 28060 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:22:26.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.353193 28060 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:22:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.357800 28060 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:22:26.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.359192 28060 hybrid_clock.cc:648] HybridClock initialized: now 1756995746359146 us; error 63 us; skew 500 ppm
14:22:26.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.360046 28060 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:22:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.367353 28060 webserver.cc:480] Webserver started at http://127.27.100.62:45031/ using document root <none> and password file <none>
14:22:26.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.368325 28060 fs_manager.cc:362] Metadata directory not provided
14:22:26.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.368603 28060 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:22:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.369081 28060 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:22:26.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.373600 28060 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-0/data/instance:
14:22:26.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5795cf6187574be49df7be30690f6ea8"
14:22:26.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:22:26 on dist-test-slave-hnxx"
14:22:26.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.374769 28060 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-0/wal/instance:
14:22:26.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5795cf6187574be49df7be30690f6ea8"
14:22:26.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:22:26 on dist-test-slave-hnxx"
14:22:26.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.382793 28060 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.000s sys 0.009s
14:22:26.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.388923 28076 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:22:26.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.390170 28060 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
14:22:26.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.390520 28060 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-0/data,/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-0/wal
14:22:26.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5795cf6187574be49df7be30690f6ea8"
14:22:26.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:22:26 on dist-test-slave-hnxx"
14:22:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.390864 28060 fs_report.cc:389] FS layout report
14:22:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:22:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-0/wal
14:22:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-0/wal
14:22:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-0/data/data
14:22:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:22:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:22:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:22:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:22:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:22:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:22:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:22:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:22:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:22:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:22:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:22:26.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.465723 28060 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:22:26.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.467272 28060 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:22:26.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.467754 28060 kserver.cc:163] Server-wide thread pool size limit: 3276
14:22:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.537334 28060 rpc_server.cc:307] RPC server started. Bound to: 127.27.100.62:38363
14:22:26.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.537403 28129 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.100.62:38363 every 8 connection(s)
14:22:26.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.540055 28060 server_base.cc:1179] Dumped server information to /tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-0/data/info.pb
14:22:26.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.541778 28048 external_mini_cluster.cc:1428] Started /tmp/dist-test-task61f5S1/build/tsan/bin/kudu as pid 28060
14:22:26.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.542335 28048 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-0/wal/instance
14:22:26.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.546340 28130 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.547857 28048 external_mini_cluster.cc:1366] Running /tmp/dist-test-task61f5S1/build/tsan/bin/kudu
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task61f5S1/build/tsan/bin/kudu
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-1/wal
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-1/data
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-1/logs
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-1/data/info.pb
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.100.61:36055
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.100.61
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.100.20:43003
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:22:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.100.62:38363,127.27.100.61:36055,127.27.100.60:40945
14:22:26.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:22:26.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37497/jwks.json
14:22:26.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --table_locations_ttl_ms=500 with env {}
14:22:26.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.559592 28130 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.100.62" port: 38363 } has no permanent_uuid. Determining permanent_uuid...
14:22:26.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.592022 28130 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.100.61" port: 36055 } has no permanent_uuid. Determining permanent_uuid...
14:22:26.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:26.594467 28077 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.100.61:36055: connect: Connection refused (error 111)
14:22:26.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:26.598387 28130 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.27.100.61:36055: Network error: Client connection negotiation failed: client connection to 127.27.100.61:36055: connect: Connection refused (error 111)
14:22:26.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.647836 28130 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.100.61" port: 36055 } attempt: 1
14:22:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:26.651705 28130 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.27.100.61:36055: Network error: Client connection negotiation failed: client connection to 127.27.100.61:36055: connect: Connection refused (error 111)
14:22:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.720114 28130 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.100.61" port: 36055 } attempt: 2
14:22:26.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:26.723953 28130 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.27.100.61:36055: Network error: Client connection negotiation failed: client connection to 127.27.100.61:36055: connect: Connection refused (error 111)
14:22:26.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.815470 28130 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.100.61" port: 36055 } attempt: 3
14:22:26.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:26.819262 28130 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.27.100.61:36055: Network error: Client connection negotiation failed: client connection to 127.27.100.61:36055: connect: Connection refused (error 111)
14:22:26.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:26.854135 28132 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:22:26.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:26.854779 28132 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:22:26.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:26.855360 28132 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:22:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:26.886509 28132 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:22:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:26.886873 28132 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:22:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:26.887212 28132 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:22:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:26.887521 28132 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:22:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:26.887787 28132 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:22:26.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:26.888021 28132 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37497/jwks.json
14:22:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.923545 28132 master_runner.cc:386] Master server non-default flags:
14:22:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.100.20:43003
14:22:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:22:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-1/data
14:22:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-1/wal
14:22:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --table_locations_ttl_ms=500
14:22:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:22:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.100.62:38363,127.27.100.61:36055,127.27.100.60:40945
14:22:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:22:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.100.61:36055
14:22:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:22:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:22:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:22:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37497/jwks.json
14:22:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-1/data/info.pb
14:22:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.100.61
14:22:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:22:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:22:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:22:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:22:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-1/logs
14:22:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:22:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:22:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:22:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:22:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:22:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:22:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 13:43:25 UTC on 5fd53c4cbb9d
14:22:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7897
14:22:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:22:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.925272 28132 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:22:26.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.927006 28132 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:22:26.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:26.937556 28141 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:26.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:26.962853 28130 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.100.61" port: 36055 } attempt: 4
14:22:26.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:26.967993 28130 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.27.100.61:36055: Network error: Client connection negotiation failed: client connection to 127.27.100.61:36055: connect: Connection refused (error 111)
14:22:27.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:27.267602 28130 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.100.61" port: 36055 } attempt: 5
14:22:27.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:27.274317 28130 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.27.100.61:36055: Network error: Client connection negotiation failed: client connection to 127.27.100.61:36055: connect: Connection refused (error 111)
14:22:27.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:27.821918 28130 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.100.61" port: 36055 } attempt: 6
14:22:27.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:27.832537 28130 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.27.100.61:36055: Network error: Client connection negotiation failed: client connection to 127.27.100.61:36055: connect: Connection refused (error 111)
14:22:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:26.938654 28142 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:28.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:28.163139 28144 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:28.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:28.166242 28143 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1224 milliseconds
14:22:28.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.166383 28132 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:22:28.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.167625 28132 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:22:28.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.170181 28132 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:22:28.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.171518 28132 hybrid_clock.cc:648] HybridClock initialized: now 1756995748171489 us; error 43 us; skew 500 ppm
14:22:28.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.172328 28132 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:22:28.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.178406 28132 webserver.cc:480] Webserver started at http://127.27.100.61:45811/ using document root <none> and password file <none>
14:22:28.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.179291 28132 fs_manager.cc:362] Metadata directory not provided
14:22:28.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.179481 28132 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:22:28.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.179862 28132 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:22:28.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.184294 28132 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-1/data/instance:
14:22:28.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "214d4c9bd4a84da7a1a7e91ce730e02e"
14:22:28.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:22:28 on dist-test-slave-hnxx"
14:22:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.185354 28132 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-1/wal/instance:
14:22:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "214d4c9bd4a84da7a1a7e91ce730e02e"
14:22:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:22:28 on dist-test-slave-hnxx"
14:22:28.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.192378 28132 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.001s
14:22:28.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.197942 28153 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:22:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.198995 28132 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
14:22:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.199285 28132 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-1/data,/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-1/wal
14:22:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "214d4c9bd4a84da7a1a7e91ce730e02e"
14:22:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:22:28 on dist-test-slave-hnxx"
14:22:28.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.199587 28132 fs_report.cc:389] FS layout report
14:22:28.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:22:28.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-1/wal
14:22:28.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-1/wal
14:22:28.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-1/data/data
14:22:28.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:22:28.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:22:28.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:22:28.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:22:28.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:22:28.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:22:28.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:22:28.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:22:28.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:22:28.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:22:28.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:22:28.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.264798 28132 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:22:28.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.266247 28132 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:22:28.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.266633 28132 kserver.cc:163] Server-wide thread pool size limit: 3276
14:22:28.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.336127 28132 rpc_server.cc:307] RPC server started. Bound to: 127.27.100.61:36055
14:22:28.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.336196 28204 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.100.61:36055 every 8 connection(s)
14:22:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.338725 28132 server_base.cc:1179] Dumped server information to /tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-1/data/info.pb
14:22:28.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.343724 28205 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:22:28.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.346103 28048 external_mini_cluster.cc:1428] Started /tmp/dist-test-task61f5S1/build/tsan/bin/kudu as pid 28132
14:22:28.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.346450 28048 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-1/wal/instance
14:22:28.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.348512 28048 external_mini_cluster.cc:1366] Running /tmp/dist-test-task61f5S1/build/tsan/bin/kudu
14:22:28.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task61f5S1/build/tsan/bin/kudu
14:22:28.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-2/wal
14:22:28.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-2/data
14:22:28.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:22:28.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-2/logs
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-2/data/info.pb
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.100.60:40945
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.100.60
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.100.20:43003
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.100.62:38363,127.27.100.61:36055,127.27.100.60:40945
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37497/jwks.json
14:22:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --table_locations_ttl_ms=500 with env {}
14:22:28.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.355758 28205 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.100.62" port: 38363 } has no permanent_uuid. Determining permanent_uuid...
14:22:28.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.379444 28205 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.100.61" port: 36055 } has no permanent_uuid. Determining permanent_uuid...
14:22:28.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.394120 28205 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.100.60" port: 40945 } has no permanent_uuid. Determining permanent_uuid...
14:22:28.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:28.396468 28156 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.100.60:40945: connect: Connection refused (error 111)
14:22:28.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:28.400707 28205 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.27.100.60:40945: Network error: Client connection negotiation failed: client connection to 127.27.100.60:40945: connect: Connection refused (error 111)
14:22:28.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.450157 28205 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.100.60" port: 40945 } attempt: 1
14:22:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:28.454715 28205 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.27.100.60:40945: Network error: Client connection negotiation failed: client connection to 127.27.100.60:40945: connect: Connection refused (error 111)
14:22:28.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.523077 28205 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.100.60" port: 40945 } attempt: 2
14:22:28.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:28.527122 28205 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.27.100.60:40945: Network error: Client connection negotiation failed: client connection to 127.27.100.60:40945: connect: Connection refused (error 111)
14:22:28.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.618525 28205 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.100.60" port: 40945 } attempt: 3
14:22:28.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:28.622659 28205 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.27.100.60:40945: Network error: Client connection negotiation failed: client connection to 127.27.100.60:40945: connect: Connection refused (error 111)
14:22:28.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:28.661326 28207 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:22:28.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:28.661993 28207 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:22:28.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:28.662569 28207 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:22:28.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:28.694808 28207 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:22:28.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:28.695184 28207 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:22:28.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:28.695500 28207 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:22:28.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:28.695789 28207 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:22:28.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:28.696112 28207 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:22:28.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:28.696393 28207 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37497/jwks.json
14:22:28.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.733242 28207 master_runner.cc:386] Master server non-default flags:
14:22:28.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:22:28.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.100.20:43003
14:22:28.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:22:28.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:22:28.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-2/data
14:22:28.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-2/wal
14:22:28.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --table_locations_ttl_ms=500
14:22:28.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:22:28.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.100.62:38363,127.27.100.61:36055,127.27.100.60:40945
14:22:28.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:22:28.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:22:28.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:22:28.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.100.60:40945
14:22:28.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:22:28.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37497/jwks.json
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-2/data/info.pb
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.100.60
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-2/logs
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 13:43:25 UTC on 5fd53c4cbb9d
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7897
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:22:28.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.734532 28207 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:22:28.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.736308 28207 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:22:28.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:28.746809 28217 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:28.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.766041 28205 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.100.60" port: 40945 } attempt: 4
14:22:28.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:28.770067 28205 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.27.100.60:40945: Network error: Client connection negotiation failed: client connection to 127.27.100.60:40945: connect: Connection refused (error 111)
14:22:28.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.893175 28130 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.100.61" port: 36055 } attempt: 7
14:22:28.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.924652 28130 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.100.60" port: 40945 } has no permanent_uuid. Determining permanent_uuid...
14:22:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:28.933176 28130 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.27.100.60:40945: Network error: Client connection negotiation failed: client connection to 127.27.100.60:40945: connect: Connection refused (error 111)
14:22:28.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:28.991740 28130 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.100.60" port: 40945 } attempt: 1
14:22:28.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:28.998188 28130 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.27.100.60:40945: Network error: Client connection negotiation failed: client connection to 127.27.100.60:40945: connect: Connection refused (error 111)
14:22:29.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:29.069509 28205 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.100.60" port: 40945 } attempt: 5
14:22:29.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:29.075815 28205 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.27.100.60:40945: Network error: Client connection negotiation failed: client connection to 127.27.100.60:40945: connect: Connection refused (error 111)
14:22:29.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:29.079785 28130 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.100.60" port: 40945 } attempt: 2
14:22:29.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:29.086382 28130 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.27.100.60:40945: Network error: Client connection negotiation failed: client connection to 127.27.100.60:40945: connect: Connection refused (error 111)
14:22:29.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:29.171922 28130 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.100.60" port: 40945 } attempt: 3
14:22:29.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:29.178480 28130 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.27.100.60:40945: Network error: Client connection negotiation failed: client connection to 127.27.100.60:40945: connect: Connection refused (error 111)
14:22:29.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:29.319041 28130 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.100.60" port: 40945 } attempt: 4
14:22:29.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:29.326188 28130 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.27.100.60:40945: Network error: Client connection negotiation failed: client connection to 127.27.100.60:40945: connect: Connection refused (error 111)
14:22:29.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:29.609794 28130 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.100.60" port: 40945 } attempt: 5
14:22:29.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:29.616617 28130 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.27.100.60:40945: Network error: Client connection negotiation failed: client connection to 127.27.100.60:40945: connect: Connection refused (error 111)
14:22:29.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:29.623361 28205 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.100.60" port: 40945 } attempt: 6
14:22:29.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:29.634447 28205 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.27.100.60:40945: Network error: Client connection negotiation failed: client connection to 127.27.100.60:40945: connect: Connection refused (error 111)
14:22:30.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:28.749171 28218 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:30.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:30.018426 28219 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1268 milliseconds
14:22:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:30.019259 28220 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:22:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:30.019516 28207 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:22:30.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:30.023517 28207 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:22:30.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:30.027089 28207 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:22:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:30.028710 28207 hybrid_clock.cc:648] HybridClock initialized: now 1756995750028656 us; error 48 us; skew 500 ppm
14:22:30.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:30.029954 28207 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:22:30.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:30.037626 28207 webserver.cc:480] Webserver started at http://127.27.100.60:33811/ using document root <none> and password file <none>
14:22:30.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:30.038618 28207 fs_manager.cc:362] Metadata directory not provided
14:22:30.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:30.038820 28207 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:22:30.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:30.039031 28207 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:22:30.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:30.039642 28207 fs_manager.cc:362] Metadata directory not provided
14:22:30.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:30.039824 28207 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:22:30.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:30.065642 28207 builtin_ntp.cc:688] coult not shutdown socket: Network error: shutdown error: Transport endpoint is not connected (error 107)
14:22:30.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Could not create new FS layout: Write-ahead log directory /tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-2/wal failed to canonicalize: Unable to canonicalize /tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824/master-2: No such file or directory (error 2)
14:22:30.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:30.169193 28130 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.100.60" port: 40945 } attempt: 6
14:22:30.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:30.175379 28130 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.27.100.60:40945: Network error: Client connection negotiation failed: client connection to 127.27.100.60:40945: connect: Connection refused (error 111)
14:22:30.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:30.695078 28205 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.100.60" port: 40945 } attempt: 7
14:22:30.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:30.701645 28205 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.27.100.60:40945: Network error: Client connection negotiation failed: client connection to 127.27.100.60:40945: connect: Connection refused (error 111)
14:22:31.104 [DEBUG - main] (MiniKuduCluster.java:195) Response: error {
code: RUNTIME_ERROR
message: "failed to start masters: Unable to start Master at index 2: /tmp/dist-test-task61f5S1/build/tsan/bin/kudu: process exited with non-zero status 1"
}
14:22:31.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:31.105126 28048 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task61f5S1/build/tsan/bin/kudu with pid 28060
14:22:31.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:22:31.126497 28048 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task61f5S1/build/tsan/bin/kudu with pid 28132
14:22:31.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:31.146574 28048 env_posix.cc:2133] Unable to access file /tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824 during walk: No such file or directory
14:22:31.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:31.146780 28048 env_posix.cc:2135] Unable to access file /tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824 during walk: No such file or directory
14:22:31.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:22:31.146977 28048 tool_action_test.cc:443] Could not delete cluster root: IO error: /tmp/dist-test-task61f5S1/test-tmp/mini-kudu-cluster6725871456096937824: One or more errors occurred
14:22:31.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==28048==WARNING: invalid path to external symbolizer!
14:22:31.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==28048==WARNING: Failed to use and restart external symbolizer!
14:22:31.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
14:22:31.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=28048)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 4 at 0x7b6800001e00 by main thread (mutexes: write M3457):
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1c0ed4)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x18f172)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libmini_oidc.so+0x21aa2)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x219ea)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0xb3e5e)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libmini_cluster.so+0x993cd)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libmini_cluster.so+0x9836b)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libmini_cluster.so+0x846c9)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libmini_cluster.so+0x84829)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x6585e7)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x654a5d)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x65412b)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x651350)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x529e81)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x529dec)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x529d5c)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x5289c5)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libkudu_tools_util.so+0x13fc1f)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libkudu_tools_util.so+0x132481)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (libkudu_tools_util.so+0x130212)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (kudu+0x645090)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #21 <null> <null> (kudu+0x6459c7)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #22 <null> <null> (kudu+0x645f0e)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #23 <null> <null> (libc.so.6+0x21bf6)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous read of size 4 at 0x7b6800001e00 by thread T5:
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1c2a2f)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (kudu+0x468cdf)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) As if synchronized via sleep:
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x48dcc2)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libgutil.so+0xb5f0a)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libkudu_util.so+0x3d7f96)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_cluster.so+0x90bce)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x9314f)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libmini_cluster.so+0x88116)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libmini_cluster.so+0x86c5d)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x652939)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x651234)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x529e81)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x529dec)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x529d5c)
14:22:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x5289c5)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x13fc1f)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x132481)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (libkudu_tools_util.so+0x130212)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x645090)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x6459c7)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (kudu+0x645f0e)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (libc.so.6+0x21bf6)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Location is heap block of size 1376 at 0x7b6800001e00 allocated by main thread:
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x498efc)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c1404)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x1900f0)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x22185)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x857e4)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x652939)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x651234)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x529e81)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x529dec)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x529d5c)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x5289c5)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13fc1f)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x132481)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x130212)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x645090)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6459c7)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x645f0e)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mutex M3457 (0x7b6800001f80) created at:
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x46cd02)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c24ef)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x1900f0)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x22185)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x857e4)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x652939)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x651234)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x529e81)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x529dec)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x529d5c)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x5289c5)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13fc1f)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x132481)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x130212)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x645090)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6459c7)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x645f0e)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T5 'sq_acceptor' (tid=28055, running) created by main thread at:
14:22:31.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x46c795)
14:22:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c278d)
14:22:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x1900f0)
14:22:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x22185)
14:22:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x857e4)
14:22:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x652939)
14:22:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x651234)
14:22:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x529e81)
14:22:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x529dec)
14:22:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x529d5c)
14:22:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x5289c5)
14:22:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13fc1f)
14:22:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x132481)
14:22:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x130212)
14:22:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x645090)
14:22:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6459c7)
14:22:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x645f0e)
14:22:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
14:22:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:22:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-task61f5S1/build/tsan/lib/libserver_process.so+0x1c0ed4)
14:22:31.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=28048)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 8 at 0x7b0c00034350 by thread T5:
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x49679f)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c2ac0)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (kudu+0x468cdf)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous read of size 4 at 0x7b0c00034350 by main thread:
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1bbcbd)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x190ca7)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x190f27)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x221cb)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x857e4)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x652939)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x651234)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x529e81)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x529dec)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x529d5c)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x5289c5)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13fc1f)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x132481)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x130212)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x645090)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6459c7)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x645f0e)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T5 'sq_acceptor' (tid=28055, running) created by main thread at:
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x46c795)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c278d)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x1900f0)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x22185)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x857e4)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x652939)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x651234)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x529e81)
14:22:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x529dec)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x529d5c)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x5289c5)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13fc1f)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x132481)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x130212)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x645090)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6459c7)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x645f0e)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-task61f5S1/build/tsan/bin/kudu+0x49679f)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=28048)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 4 at 0x7b6800001800 by main thread (mutexes: write M3454):
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1c0ed4)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x18f172)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libmini_oidc.so+0x21ac5)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x219ea)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0xb3e5e)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libmini_cluster.so+0x993cd)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libmini_cluster.so+0x9836b)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libmini_cluster.so+0x846c9)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libmini_cluster.so+0x84829)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x6585e7)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x654a5d)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x65412b)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x651350)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x529e81)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x529dec)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x529d5c)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x5289c5)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libkudu_tools_util.so+0x13fc1f)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libkudu_tools_util.so+0x132481)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (libkudu_tools_util.so+0x130212)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (kudu+0x645090)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #21 <null> <null> (kudu+0x6459c7)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #22 <null> <null> (kudu+0x645f0e)
14:22:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #23 <null> <null> (libc.so.6+0x21bf6)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous read of size 4 at 0x7b6800001800 by thread T4:
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1c2a2f)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (kudu+0x468cdf)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) As if synchronized via sleep:
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x48d6fd)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c0f79)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18f172)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x21aa2)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_oidc.so+0x219ea)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libmini_cluster.so+0xb3e5e)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libmini_cluster.so+0x993cd)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libmini_cluster.so+0x9836b)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libmini_cluster.so+0x846c9)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (libmini_cluster.so+0x84829)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x6585e7)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x654a5d)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x65412b)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x651350)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x529e81)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x529dec)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x529d5c)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x5289c5)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libkudu_tools_util.so+0x13fc1f)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (libkudu_tools_util.so+0x132481)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (libkudu_tools_util.so+0x130212)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #21 <null> <null> (kudu+0x645090)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #22 <null> <null> (kudu+0x6459c7)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #23 <null> <null> (kudu+0x645f0e)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #24 <null> <null> (libc.so.6+0x21bf6)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Location is heap block of size 1376 at 0x7b6800001800 allocated by main thread:
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x498efc)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c1404)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x1900f0)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x21dd0)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x857e4)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x652939)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x651234)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x529e81)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x529dec)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x529d5c)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x5289c5)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13fc1f)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x132481)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x130212)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x645090)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6459c7)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x645f0e)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mutex M3454 (0x7b6800001980) created at:
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x46cd02)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c24ef)
14:22:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x1900f0)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x21dd0)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x857e4)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x652939)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x651234)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x529e81)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x529dec)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x529d5c)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x5289c5)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13fc1f)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x132481)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x130212)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x645090)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6459c7)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x645f0e)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T4 'sq_acceptor' (tid=28054, running) created by main thread at:
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x46c795)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c278d)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x1900f0)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x21dd0)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x857e4)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x652939)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x651234)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x529e81)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x529dec)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x529d5c)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x5289c5)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13fc1f)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x132481)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x130212)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x645090)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6459c7)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x645f0e)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-task61f5S1/build/tsan/lib/libserver_process.so+0x1c0ed4)
14:22:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
14:22:31.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
14:22:31.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=28048)
14:22:31.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 8 at 0x7b0c00031aa0 by thread T4:
14:22:31.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x49679f)
14:22:31.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c2ac0)
14:22:31.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (kudu+0x468cdf)
14:22:31.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:22:31.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous read of size 4 at 0x7b0c00031aa0 by main thread:
14:22:31.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1bbcbd)
14:22:31.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x190ca7)
14:22:31.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x190f27)
14:22:31.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x21e21)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x857e4)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x652939)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x651234)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x529e81)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x529dec)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x529d5c)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x5289c5)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13fc1f)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x132481)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x130212)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x645090)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6459c7)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x645f0e)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T4 'sq_acceptor' (tid=28054, running) created by main thread at:
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x46c795)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c278d)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x1900f0)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x21dd0)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x857e4)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x652939)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x651234)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x529e81)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x529dec)
14:22:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x529d5c)
14:22:31.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x5289c5)
14:22:31.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13fc1f)
14:22:31.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x132481)
14:22:31.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x130212)
14:22:31.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x645090)
14:22:31.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6459c7)
14:22:31.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x645f0e)
14:22:31.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
14:22:31.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:22:31.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-task61f5S1/build/tsan/bin/kudu+0x49679f)
14:22:31.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
14:22:31.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-04T14:22:31Z chronyd exiting
14:22:32.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ThreadSanitizer: reported 4 warnings
14:22:32.219 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduClient.testConcurrentFlush: failed attempt 1
java.io.IOException: failed to start masters: Unable to start Master at index 2: /tmp/dist-test-task61f5S1/build/tsan/bin/kudu: process exited with non-zero status 1
at org.apache.kudu.test.cluster.MiniKuduCluster.sendRequestToCluster(MiniKuduCluster.java:199) ~[kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster.start(MiniKuduCluster.java:266) ~[kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster.access$300(MiniKuduCluster.java:85) ~[kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster$MiniKuduClusterBuilder.build(MiniKuduCluster.java:897) ~[kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:156) ~[kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.io.IOException: failed to start masters: Unable to start Master at index 2: /tmp/dist-test-task61f5S1/build/tsan/bin/kudu: process exited with non-zero status 1
at org.apache.kudu.test.cluster.MiniKuduCluster.sendRequestToCluster(MiniKuduCluster.java:199)
at org.apache.kudu.test.cluster.MiniKuduCluster.start(MiniKuduCluster.java:266)
at org.apache.kudu.test.cluster.MiniKuduCluster.access$300(MiniKuduCluster.java:85)
at org.apache.kudu.test.cluster.MiniKuduCluster$MiniKuduClusterBuilder.build(MiniKuduCluster.java:897)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:156)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)