22:00:26.977 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
22:00:26.978 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task8UTc2V/test-tmp
22:00:26.978 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
22:00:26.978 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task8UTc2V/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
22:00:26.979 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
22:00:27.310 [DEBUG - main] (MiniKuduCluster.java:195) Response:
22:00:27.310 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
22:00:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:27.313333 2056 mini_oidc.cc:152] Starting JWKS server
22:00:27.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:27.316910 2056 webserver.cc:460] Webserver started at http://127.0.0.1:35101/ using document root <none> and password file <none>
22:00:27.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:27.317341 2056 mini_oidc.cc:189] Starting OIDC Discovery server
22:00:27.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:27.319129 2056 webserver.cc:460] Webserver started at http://127.0.0.1:43433/ using document root <none> and password file <none>
22:00:27.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T22:00:27Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
22:00:27.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T22:00:27Z Disabled control of system clock
22:00:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:27.390223 2056 external_mini_cluster.cc:1326] Running /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu
22:00:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu
22:00:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-0/wal
22:00:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-0/data
22:00:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:00:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:00:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:00:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:00:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:00:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:00:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-0/logs
22:00:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-0/data/info.pb
22:00:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:00:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:00:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:00:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:00:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:00:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:00:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:00:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:00:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:00:27.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:00:27.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:00:27.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:00:27.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.2.62:39037
22:00:27.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.2.62
22:00:27.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:00:27.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.2.20:39983
22:00:27.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:00:27.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:00:27.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:00:27.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:00:27.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.2.62:39037,127.2.2.61:43083,127.2.2.60:44059
22:00:27.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:00:27.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35101/jwks.json with env {}
22:00:27.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:27.695261 2069 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:00:27.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:27.695830 2069 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:00:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:27.696377 2069 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:00:27.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:27.727746 2069 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:00:27.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:27.728121 2069 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:00:27.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:27.728363 2069 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:00:27.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:27.728564 2069 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:00:27.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:27.728755 2069 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:00:27.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:27.728907 2069 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35101/jwks.json
22:00:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:27.764173 2069 master_runner.cc:386] Master server non-default flags:
22:00:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:00:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.2.20:39983
22:00:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:00:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:00:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-0/data
22:00:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-0/wal
22:00:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:00:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.2.62:39037,127.2.2.61:43083,127.2.2.60:44059
22:00:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:00:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:00:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:00:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.2.62:39037
22:00:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:00:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:00:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:00:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35101/jwks.json
22:00:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:00:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:00:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-0/data/info.pb
22:00:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.2.62
22:00:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:00:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:00:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:00:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:00:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:00:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:00:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-0/logs
22:00:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:00:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:00:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:00:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:00:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
22:00:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:00:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
22:00:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
22:00:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:00:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:27.765383 2069 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:00:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:27.766940 2069 file_cache.cc:492] Constructed file cache file cache with capacity 419430
22:00:27.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:27.782467 2075 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:00:28.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:27.783123 2078 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:00:28.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:27.783566 2076 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:00:28.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:27.784838 2069 server_base.cc:1034] running on GCE node
22:00:28.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:28.885115 2069 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:00:28.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:28.887816 2069 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:00:28.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:28.889158 2069 hybrid_clock.cc:648] HybridClock initialized: now 1714687228889121 us; error 41 us; skew 500 ppm
22:00:28.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:28.889976 2069 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:00:28.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:28.896248 2069 webserver.cc:460] Webserver started at http://127.2.2.62:44567/ using document root <none> and password file <none>
22:00:28.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:28.897218 2069 fs_manager.cc:359] Metadata directory not provided
22:00:28.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:28.897403 2069 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:00:28.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:28.897802 2069 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:00:28.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:28.902734 2069 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-0/data/instance:
22:00:28.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "445dc5aee8934a8aadf1de4114189469"
22:00:28.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 22:00:28 on dist-test-slave-ntzt"
22:00:28.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:28.903867 2069 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-0/wal/instance:
22:00:28.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "445dc5aee8934a8aadf1de4114189469"
22:00:28.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 22:00:28 on dist-test-slave-ntzt"
22:00:28.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:28.910897 2069 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.004s sys 0.002s
22:00:28.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:28.916309 2085 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:00:28.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:28.917475 2069 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
22:00:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:28.917774 2069 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-0/data,/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-0/wal
22:00:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "445dc5aee8934a8aadf1de4114189469"
22:00:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 22:00:28 on dist-test-slave-ntzt"
22:00:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:28.918133 2069 fs_report.cc:383] FS layout report
22:00:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:00:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-0/wal
22:00:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-0/wal
22:00:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-0/data/data
22:00:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:00:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:00:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:00:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:00:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:00:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:00:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:00:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:00:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:00:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:00:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:00:28.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:28.966171 2069 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
22:00:28.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:28.967864 2069 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:00:28.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:28.968477 2069 kserver.cc:163] Server-wide thread pool size limit: 3276
22:00:29.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:29.040252 2069 rpc_server.cc:307] RPC server started. Bound to: 127.2.2.62:39037
22:00:29.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:29.040378 2138 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.2.62:39037 every 8 connection(s)
22:00:29.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:29.043017 2069 server_base.cc:1166] Dumped server information to /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-0/data/info.pb
22:00:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:29.045979 2056 external_mini_cluster.cc:1388] Started /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu as pid 2069
22:00:29.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:29.046531 2056 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-0/wal/instance
22:00:29.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:29.049453 2139 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:00:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:29.052098 2056 external_mini_cluster.cc:1326] Running /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu
22:00:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu
22:00:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-1/wal
22:00:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-1/data
22:00:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:00:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:00:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:00:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:00:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:00:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:00:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-1/logs
22:00:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-1/data/info.pb
22:00:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:00:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:00:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:00:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:00:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:00:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:00:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:00:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:00:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:00:29.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:00:29.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:00:29.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:00:29.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.2.61:43083
22:00:29.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.2.61
22:00:29.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:00:29.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.2.20:39983
22:00:29.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:00:29.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:00:29.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:00:29.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:00:29.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.2.62:39037,127.2.2.61:43083,127.2.2.60:44059
22:00:29.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:00:29.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35101/jwks.json with env {}
22:00:29.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:29.064543 2139 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.2.62" port: 39037 } has no permanent_uuid. Determining permanent_uuid...
22:00:29.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:29.090519 2139 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } has no permanent_uuid. Determining permanent_uuid...
22:00:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:29.093058 2086 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.2.61:43083: connect: Connection refused (error 111)
22:00:29.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:29.097436 2139 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.2.2.61:43083: Network error: Client connection negotiation failed: client connection to 127.2.2.61:43083: connect: Connection refused (error 111)
22:00:29.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:29.147042 2139 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } attempt: 1
22:00:29.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:29.151162 2139 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.2.2.61:43083: Network error: Client connection negotiation failed: client connection to 127.2.2.61:43083: connect: Connection refused (error 111)
22:00:29.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:29.219679 2139 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } attempt: 2
22:00:29.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:29.223855 2139 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.2.2.61:43083: Network error: Client connection negotiation failed: client connection to 127.2.2.61:43083: connect: Connection refused (error 111)
22:00:29.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:29.315397 2139 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } attempt: 3
22:00:29.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:29.319401 2139 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.2.2.61:43083: Network error: Client connection negotiation failed: client connection to 127.2.2.61:43083: connect: Connection refused (error 111)
22:00:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:29.371927 2141 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:00:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:29.372550 2141 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:00:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:29.373061 2141 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:00:29.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:29.404417 2141 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:00:29.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:29.404716 2141 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:00:29.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:29.404927 2141 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:00:29.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:29.405107 2141 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:00:29.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:29.405287 2141 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:00:29.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:29.405438 2141 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35101/jwks.json
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:29.440757 2141 master_runner.cc:386] Master server non-default flags:
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.2.20:39983
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-1/data
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-1/wal
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.2.62:39037,127.2.2.61:43083,127.2.2.60:44059
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.2.61:43083
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35101/jwks.json
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-1/data/info.pb
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.2.61
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-1/logs
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:00:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
22:00:29.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:00:29.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
22:00:29.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
22:00:29.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:00:29.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:29.442039 2141 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:00:29.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:29.443655 2141 file_cache.cc:492] Constructed file cache file cache with capacity 419430
22:00:29.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:29.459828 2153 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:00:29.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:29.462877 2139 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } attempt: 4
22:00:29.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:29.466701 2139 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.2.2.61:43083: Network error: Client connection negotiation failed: client connection to 127.2.2.61:43083: connect: Connection refused (error 111)
22:00:29.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:29.766227 2139 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } attempt: 5
22:00:29.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:29.770753 2139 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.2.2.61:43083: Network error: Client connection negotiation failed: client connection to 127.2.2.61:43083: connect: Connection refused (error 111)
22:00:30.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.318490 2139 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } attempt: 6
22:00:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:30.326265 2139 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.2.2.61:43083: Network error: Client connection negotiation failed: client connection to 127.2.2.61:43083: connect: Connection refused (error 111)
22:00:30.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:29.460088 2150 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:00:30.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:29.460515 2151 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:00:30.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:29.461081 2141 server_base.cc:1034] running on GCE node
22:00:30.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.584666 2141 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:00:30.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.587579 2141 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:00:30.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.588985 2141 hybrid_clock.cc:648] HybridClock initialized: now 1714687230588953 us; error 43 us; skew 500 ppm
22:00:30.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.589826 2141 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:00:30.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.601438 2141 webserver.cc:460] Webserver started at http://127.2.2.61:38697/ using document root <none> and password file <none>
22:00:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.602380 2141 fs_manager.cc:359] Metadata directory not provided
22:00:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.602577 2141 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:00:30.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.602985 2141 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:00:30.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.608106 2141 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-1/data/instance:
22:00:30.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd"
22:00:30.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 22:00:30 on dist-test-slave-ntzt"
22:00:30.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.609273 2141 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-1/wal/instance:
22:00:30.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd"
22:00:30.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 22:00:30 on dist-test-slave-ntzt"
22:00:30.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.616555 2141 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
22:00:30.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.622164 2162 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:00:30.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.623206 2141 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
22:00:30.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.623556 2141 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-1/data,/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-1/wal
22:00:30.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd"
22:00:30.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 22:00:30 on dist-test-slave-ntzt"
22:00:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.623919 2141 fs_report.cc:383] FS layout report
22:00:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:00:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-1/wal
22:00:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-1/wal
22:00:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-1/data/data
22:00:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:00:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:00:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:00:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:00:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:00:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:00:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:00:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:00:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:00:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:00:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:00:30.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.672780 2141 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
22:00:30.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.674356 2141 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:00:30.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.674854 2141 kserver.cc:163] Server-wide thread pool size limit: 3276
22:00:30.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.748086 2141 rpc_server.cc:307] RPC server started. Bound to: 127.2.2.61:43083
22:00:30.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.748178 2214 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.2.61:43083 every 8 connection(s)
22:00:30.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.750977 2141 server_base.cc:1166] Dumped server information to /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-1/data/info.pb
22:00:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.756362 2215 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:00:30.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.760530 2056 external_mini_cluster.cc:1388] Started /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu as pid 2141
22:00:30.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.760885 2056 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-1/wal/instance
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.762918 2056 external_mini_cluster.cc:1326] Running /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-2/wal
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-2/data
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-2/logs
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-2/data/info.pb
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.2.60:44059
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.2.60
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.2.20:39983
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:00:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:00:30.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:00:30.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:00:30.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.2.62:39037,127.2.2.61:43083,127.2.2.60:44059
22:00:30.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:00:30.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35101/jwks.json with env {}
22:00:30.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.768391 2215 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.2.62" port: 39037 } has no permanent_uuid. Determining permanent_uuid...
22:00:30.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.791988 2215 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } has no permanent_uuid. Determining permanent_uuid...
22:00:30.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.805887 2215 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } has no permanent_uuid. Determining permanent_uuid...
22:00:30.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:30.808097 2166 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.2.60:44059: connect: Connection refused (error 111)
22:00:30.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:30.812350 2215 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.2.2.60:44059: Network error: Client connection negotiation failed: client connection to 127.2.2.60:44059: connect: Connection refused (error 111)
22:00:30.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.861925 2215 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } attempt: 1
22:00:30.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:30.865998 2215 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.2.2.60:44059: Network error: Client connection negotiation failed: client connection to 127.2.2.60:44059: connect: Connection refused (error 111)
22:00:30.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:30.934501 2215 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } attempt: 2
22:00:30.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:30.938462 2215 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.2.2.60:44059: Network error: Client connection negotiation failed: client connection to 127.2.2.60:44059: connect: Connection refused (error 111)
22:00:31.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:31.029951 2215 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } attempt: 3
22:00:31.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:31.033943 2215 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.2.2.60:44059: Network error: Client connection negotiation failed: client connection to 127.2.2.60:44059: connect: Connection refused (error 111)
22:00:31.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:31.079720 2217 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:00:31.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:31.080346 2217 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:00:31.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:31.080837 2217 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:00:31.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:31.112188 2217 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:00:31.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:31.112510 2217 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:00:31.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:31.112709 2217 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:00:31.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:31.112915 2217 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:00:31.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:31.113114 2217 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:00:31.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:31.113262 2217 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35101/jwks.json
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:31.148864 2217 master_runner.cc:386] Master server non-default flags:
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.2.20:39983
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-2/data
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-2/wal
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.2.62:39037,127.2.2.61:43083,127.2.2.60:44059
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.2.60:44059
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35101/jwks.json
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-2/data/info.pb
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.2.60
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-2/logs
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
22:00:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
22:00:31.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:00:31.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:31.150111 2217 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:00:31.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:31.151716 2217 file_cache.cc:492] Constructed file cache file cache with capacity 419430
22:00:31.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:31.177408 2215 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } attempt: 4
22:00:31.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:31.181996 2215 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.2.2.60:44059: Network error: Client connection negotiation failed: client connection to 127.2.2.60:44059: connect: Connection refused (error 111)
22:00:31.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:31.387002 2139 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } attempt: 7
22:00:31.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:31.401304 2139 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } has no permanent_uuid. Determining permanent_uuid...
22:00:31.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:31.408236 2139 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.2.2.60:44059: Network error: Client connection negotiation failed: client connection to 127.2.2.60:44059: connect: Connection refused (error 111)
22:00:31.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:31.466712 2139 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } attempt: 1
22:00:31.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:31.472014 2139 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.2.2.60:44059: Network error: Client connection negotiation failed: client connection to 127.2.2.60:44059: connect: Connection refused (error 111)
22:00:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:31.481451 2215 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } attempt: 5
22:00:31.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:31.486191 2215 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.2.2.60:44059: Network error: Client connection negotiation failed: client connection to 127.2.2.60:44059: connect: Connection refused (error 111)
22:00:31.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:31.553439 2139 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } attempt: 2
22:00:31.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:31.558941 2139 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.2.2.60:44059: Network error: Client connection negotiation failed: client connection to 127.2.2.60:44059: connect: Connection refused (error 111)
22:00:31.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:31.644395 2139 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } attempt: 3
22:00:31.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:31.649899 2139 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.2.2.60:44059: Network error: Client connection negotiation failed: client connection to 127.2.2.60:44059: connect: Connection refused (error 111)
22:00:31.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:31.790331 2139 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } attempt: 4
22:00:31.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:31.795647 2139 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.2.2.60:44059: Network error: Client connection negotiation failed: client connection to 127.2.2.60:44059: connect: Connection refused (error 111)
22:00:32.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.033846 2215 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } attempt: 6
22:00:32.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:32.041615 2215 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.2.2.60:44059: Network error: Client connection negotiation failed: client connection to 127.2.2.60:44059: connect: Connection refused (error 111)
22:00:32.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.079095 2139 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } attempt: 5
22:00:32.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:32.083537 2139 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.2.2.60:44059: Network error: Client connection negotiation failed: client connection to 127.2.2.60:44059: connect: Connection refused (error 111)
22:00:32.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:31.167171 2228 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:00:32.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:31.167140 2227 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:00:32.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:31.169517 2230 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:00:32.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:32.293085 2229 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
22:00:32.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.293187 2217 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
22:00:32.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.298692 2217 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:00:32.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.301867 2217 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:00:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.303311 2217 hybrid_clock.cc:648] HybridClock initialized: now 1714687232303266 us; error 59 us; skew 500 ppm
22:00:32.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.304152 2217 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:00:32.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.311570 2217 webserver.cc:460] Webserver started at http://127.2.2.60:41113/ using document root <none> and password file <none>
22:00:32.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.312690 2217 fs_manager.cc:359] Metadata directory not provided
22:00:32.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.312923 2217 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:00:32.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.313374 2217 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:00:32.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.318439 2217 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-2/data/instance:
22:00:32.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d76f973fad09443e80a402f2ad4e099d"
22:00:32.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 22:00:32 on dist-test-slave-ntzt"
22:00:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.319545 2217 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-2/wal/instance:
22:00:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d76f973fad09443e80a402f2ad4e099d"
22:00:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 22:00:32 on dist-test-slave-ntzt"
22:00:32.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.326896 2217 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.003s
22:00:32.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.332582 2242 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:00:32.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.333683 2217 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
22:00:32.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.334040 2217 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-2/data,/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-2/wal
22:00:32.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d76f973fad09443e80a402f2ad4e099d"
22:00:32.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 22:00:32 on dist-test-slave-ntzt"
22:00:32.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.334424 2217 fs_report.cc:383] FS layout report
22:00:32.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:00:32.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-2/wal
22:00:32.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-2/wal
22:00:32.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-2/data/data
22:00:32.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:00:32.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:00:32.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:00:32.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:00:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:00:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:00:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:00:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:00:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:00:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:00:32.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:00:32.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.395520 2217 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
22:00:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.397099 2217 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:00:32.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.397516 2217 kserver.cc:163] Server-wide thread pool size limit: 3276
22:00:32.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.468252 2217 rpc_server.cc:307] RPC server started. Bound to: 127.2.2.60:44059
22:00:32.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.468439 2294 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.2.60:44059 every 8 connection(s)
22:00:32.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.471242 2217 server_base.cc:1166] Dumped server information to /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-2/data/info.pb
22:00:32.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.473088 2056 external_mini_cluster.cc:1388] Started /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu as pid 2217
22:00:32.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.473625 2056 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/master-2/wal/instance
22:00:32.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.478094 2295 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:00:32.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.497563 2295 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.2.62" port: 39037 } has no permanent_uuid. Determining permanent_uuid...
22:00:32.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.523695 2295 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } has no permanent_uuid. Determining permanent_uuid...
22:00:32.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.533666 2295 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } has no permanent_uuid. Determining permanent_uuid...
22:00:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.549079 2295 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d: Bootstrap starting.
22:00:32.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.554956 2295 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d: Neither blocks nor log segments found. Creating new log.
22:00:32.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.556989 2295 log.cc:824] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d: Log is configured to *not* fsync() on all Append() calls
22:00:32.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.561353 2295 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d: No bootstrap required, opened a new log
22:00:32.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.584064 2295 raft_consensus.cc:348] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "445dc5aee8934a8aadf1de4114189469" member_type: VOTER last_known_addr { host: "127.2.2.62" port: 39037 } } peers { permanent_uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd" member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } } peers { permanent_uuid: "d76f973fad09443e80a402f2ad4e099d" member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } }
22:00:32.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.584832 2295 raft_consensus.cc:374] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
22:00:32.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.585162 2295 raft_consensus.cc:729] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d76f973fad09443e80a402f2ad4e099d, State: Initialized, Role: FOLLOWER
22:00:32.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.586050 2295 consensus_queue.cc:260] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "445dc5aee8934a8aadf1de4114189469" member_type: VOTER last_known_addr { host: "127.2.2.62" port: 39037 } } peers { permanent_uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd" member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } } peers { permanent_uuid: "d76f973fad09443e80a402f2ad4e099d" member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } }
22:00:32.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.589008 2305 sys_catalog.cc:455] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "445dc5aee8934a8aadf1de4114189469" member_type: VOTER last_known_addr { host: "127.2.2.62" port: 39037 } } peers { permanent_uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd" member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } } peers { permanent_uuid: "d76f973fad09443e80a402f2ad4e099d" member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } } }
22:00:32.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.589959 2305 sys_catalog.cc:458] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [sys.catalog]: This master's current role is: FOLLOWER
22:00:32.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.590937 2295 sys_catalog.cc:564] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [sys.catalog]: configured and running, proceeding with master startup.
22:00:32.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:32.615839 2316 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:00:32.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:32.616238 2316 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:00:32.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.635943 2139 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } attempt: 6
22:00:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.611945 2217 master_runner.cc:186] Error getting master registration for 127.2.2.62:39037: OK, instance_id { permanent_uuid: "445dc5aee8934a8aadf1de4114189469" instance_seqno: 1714687229020296 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
22:00:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.641101 2217 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:00:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:32.652658 2248 tablet.cc:2369] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d: Can't schedule compaction. Clean time has not been advanced past its initial value.
22:00:32.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.657110 2139 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469: Bootstrap starting.
22:00:32.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.663260 2139 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469: Neither blocks nor log segments found. Creating new log.
22:00:32.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.665246 2139 log.cc:824] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469: Log is configured to *not* fsync() on all Append() calls
22:00:32.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.670039 2139 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469: No bootstrap required, opened a new log
22:00:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.692087 2139 raft_consensus.cc:348] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "445dc5aee8934a8aadf1de4114189469" member_type: VOTER last_known_addr { host: "127.2.2.62" port: 39037 } } peers { permanent_uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd" member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } } peers { permanent_uuid: "d76f973fad09443e80a402f2ad4e099d" member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } }
22:00:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.693104 2139 raft_consensus.cc:374] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
22:00:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.693434 2139 raft_consensus.cc:729] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 445dc5aee8934a8aadf1de4114189469, State: Initialized, Role: FOLLOWER
22:00:32.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.694392 2139 consensus_queue.cc:260] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "445dc5aee8934a8aadf1de4114189469" member_type: VOTER last_known_addr { host: "127.2.2.62" port: 39037 } } peers { permanent_uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd" member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } } peers { permanent_uuid: "d76f973fad09443e80a402f2ad4e099d" member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } }
22:00:32.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.698551 2320 sys_catalog.cc:455] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "445dc5aee8934a8aadf1de4114189469" member_type: VOTER last_known_addr { host: "127.2.2.62" port: 39037 } } peers { permanent_uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd" member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } } peers { permanent_uuid: "d76f973fad09443e80a402f2ad4e099d" member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } } }
22:00:32.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.699474 2320 sys_catalog.cc:458] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469 [sys.catalog]: This master's current role is: FOLLOWER
22:00:32.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.700630 2139 sys_catalog.cc:564] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469 [sys.catalog]: configured and running, proceeding with master startup.
22:00:32.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:32.728142 2091 tablet.cc:2369] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469: Can't schedule compaction. Clean time has not been advanced past its initial value.
22:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:32.733747 2332 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:00:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:32.734115 2332 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:00:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.749822 2305 raft_consensus.cc:482] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
22:00:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.750411 2305 raft_consensus.cc:504] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "445dc5aee8934a8aadf1de4114189469" member_type: VOTER last_known_addr { host: "127.2.2.62" port: 39037 } } peers { permanent_uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd" member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } } peers { permanent_uuid: "d76f973fad09443e80a402f2ad4e099d" member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } }
22:00:32.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.753520 2305 leader_election.cc:288] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 445dc5aee8934a8aadf1de4114189469 (127.2.2.62:39037), e0dfa7f9a21b4c5aa45380d7fd8529fd (127.2.2.61:43083)
22:00:32.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.756160 2112 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d76f973fad09443e80a402f2ad4e099d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "445dc5aee8934a8aadf1de4114189469" is_pre_election: true
22:00:32.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.757162 2112 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d76f973fad09443e80a402f2ad4e099d in term 0.
22:00:32.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.759146 2246 leader_election.cc:302] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 445dc5aee8934a8aadf1de4114189469, d76f973fad09443e80a402f2ad4e099d; no voters:
22:00:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.760711 2305 raft_consensus.cc:2781] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [term 0 FOLLOWER]: Leader pre-election won for term 1
22:00:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.761085 2305 raft_consensus.cc:482] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
22:00:32.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.761401 2305 raft_consensus.cc:3037] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [term 0 FOLLOWER]: Advancing to term 1
22:00:32.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.773733 2305 raft_consensus.cc:504] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "445dc5aee8934a8aadf1de4114189469" member_type: VOTER last_known_addr { host: "127.2.2.62" port: 39037 } } peers { permanent_uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd" member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } } peers { permanent_uuid: "d76f973fad09443e80a402f2ad4e099d" member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } }
22:00:32.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.788991 2305 leader_election.cc:288] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [CANDIDATE]: Term 1 election: Requested vote from peers 445dc5aee8934a8aadf1de4114189469 (127.2.2.62:39037), e0dfa7f9a21b4c5aa45380d7fd8529fd (127.2.2.61:43083)
22:00:32.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.791464 2112 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d76f973fad09443e80a402f2ad4e099d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "445dc5aee8934a8aadf1de4114189469"
22:00:32.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.792104 2112 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469 [term 0 FOLLOWER]: Advancing to term 1
22:00:32.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.801476 2112 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d76f973fad09443e80a402f2ad4e099d in term 1.
22:00:32.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.803026 2246 leader_election.cc:302] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 445dc5aee8934a8aadf1de4114189469, d76f973fad09443e80a402f2ad4e099d; no voters:
22:00:32.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.803925 2305 raft_consensus.cc:2781] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [term 1 FOLLOWER]: Leader election won for term 1
22:00:32.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.734197 2069 master_runner.cc:186] Error getting master registration for 127.2.2.61:43083: OK, instance_id { permanent_uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd" instance_seqno: 1714687230727754 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
22:00:32.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.807474 2069 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:00:32.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.816558 2305 raft_consensus.cc:686] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [term 1 LEADER]: Becoming Leader. State: Replica: d76f973fad09443e80a402f2ad4e099d, State: Running, Role: LEADER
22:00:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.817641 2305 consensus_queue.cc:237] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "445dc5aee8934a8aadf1de4114189469" member_type: VOTER last_known_addr { host: "127.2.2.62" port: 39037 } } peers { permanent_uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd" member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } } peers { permanent_uuid: "d76f973fad09443e80a402f2ad4e099d" member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } }
22:00:32.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.827025 2334 sys_catalog.cc:455] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [sys.catalog]: SysCatalogTable state changed. Reason: New leader d76f973fad09443e80a402f2ad4e099d. Latest consensus state: current_term: 1 leader_uuid: "d76f973fad09443e80a402f2ad4e099d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "445dc5aee8934a8aadf1de4114189469" member_type: VOTER last_known_addr { host: "127.2.2.62" port: 39037 } } peers { permanent_uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd" member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } } peers { permanent_uuid: "d76f973fad09443e80a402f2ad4e099d" member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } } }
22:00:32.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.827937 2334 sys_catalog.cc:458] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [sys.catalog]: This master's current role is: LEADER
22:00:32.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.837589 2336 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
22:00:32.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.844683 2336 catalog_manager.cc:1484] Initializing Kudu cluster ID...
22:00:32.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.876567 2112 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469 [term 1 FOLLOWER]: Refusing update from remote peer d76f973fad09443e80a402f2ad4e099d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
22:00:32.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.878444 2305 consensus_queue.cc:1035] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [LEADER]: Connected to new peer: Peer: permanent_uuid: "445dc5aee8934a8aadf1de4114189469" member_type: VOTER last_known_addr { host: "127.2.2.62" port: 39037 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
22:00:32.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.908208 2320 sys_catalog.cc:455] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d76f973fad09443e80a402f2ad4e099d. Latest consensus state: current_term: 1 leader_uuid: "d76f973fad09443e80a402f2ad4e099d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "445dc5aee8934a8aadf1de4114189469" member_type: VOTER last_known_addr { host: "127.2.2.62" port: 39037 } } peers { permanent_uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd" member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } } peers { permanent_uuid: "d76f973fad09443e80a402f2ad4e099d" member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } } }
22:00:32.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.912088 2320 sys_catalog.cc:458] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469 [sys.catalog]: This master's current role is: FOLLOWER
22:00:32.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.920397 2337 mvcc.cc:205] Tried to move back new op lower bound from 7023358905831407616 to 7023358905632391168. Current Snapshot: MvccSnapshot[applied={T|T < 7023358905831407616}]
22:00:32.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.935070 2334 sys_catalog.cc:455] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d76f973fad09443e80a402f2ad4e099d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "445dc5aee8934a8aadf1de4114189469" member_type: VOTER last_known_addr { host: "127.2.2.62" port: 39037 } } peers { permanent_uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd" member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } } peers { permanent_uuid: "d76f973fad09443e80a402f2ad4e099d" member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } } }
22:00:32.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.935892 2334 sys_catalog.cc:458] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [sys.catalog]: This master's current role is: LEADER
22:00:32.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.942389 2320 sys_catalog.cc:455] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d76f973fad09443e80a402f2ad4e099d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "445dc5aee8934a8aadf1de4114189469" member_type: VOTER last_known_addr { host: "127.2.2.62" port: 39037 } } peers { permanent_uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd" member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } } peers { permanent_uuid: "d76f973fad09443e80a402f2ad4e099d" member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } } }
22:00:32.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.943188 2320 sys_catalog.cc:458] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469 [sys.catalog]: This master's current role is: FOLLOWER
22:00:32.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.947007 2336 catalog_manager.cc:1347] Generated new cluster ID: 3cf0617eb5914ffcb5bd2cb55bae2918
22:00:32.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.947325 2336 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
22:00:32.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.986670 2336 catalog_manager.cc:1370] Generated new certificate authority record
22:00:32.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:32.989449 2336 catalog_manager.cc:1504] Loading token signing keys...
22:00:33.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.030722 2336 catalog_manager.cc:5857] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d: Generated new TSK 0
22:00:33.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.031906 2336 catalog_manager.cc:1514] Initializing in-progress tserver states...
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.056957 2056 external_mini_cluster.cc:1326] Running /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-0/wal
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-0/data
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-0/logs
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-0/data/info.pb
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.2.1:0
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.2.1
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.2.1
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.2.62:39037,127.2.2.61:43083,127.2.2.60:44059
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.2.20:39983
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:00:33.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
22:00:33.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.102018 2215 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } attempt: 7
22:00:33.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.158370 2215 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e0dfa7f9a21b4c5aa45380d7fd8529fd: Bootstrap starting.
22:00:33.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.177704 2215 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e0dfa7f9a21b4c5aa45380d7fd8529fd: Neither blocks nor log segments found. Creating new log.
22:00:33.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.180377 2215 log.cc:824] T 00000000000000000000000000000000 P e0dfa7f9a21b4c5aa45380d7fd8529fd: Log is configured to *not* fsync() on all Append() calls
22:00:33.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.197602 2215 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e0dfa7f9a21b4c5aa45380d7fd8529fd: No bootstrap required, opened a new log
22:00:33.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.269518 2215 raft_consensus.cc:348] T 00000000000000000000000000000000 P e0dfa7f9a21b4c5aa45380d7fd8529fd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "445dc5aee8934a8aadf1de4114189469" member_type: VOTER last_known_addr { host: "127.2.2.62" port: 39037 } } peers { permanent_uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd" member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } } peers { permanent_uuid: "d76f973fad09443e80a402f2ad4e099d" member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } }
22:00:33.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.271925 2215 raft_consensus.cc:374] T 00000000000000000000000000000000 P e0dfa7f9a21b4c5aa45380d7fd8529fd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
22:00:33.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.272888 2215 raft_consensus.cc:729] T 00000000000000000000000000000000 P e0dfa7f9a21b4c5aa45380d7fd8529fd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e0dfa7f9a21b4c5aa45380d7fd8529fd, State: Initialized, Role: FOLLOWER
22:00:33.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.275327 2215 consensus_queue.cc:260] T 00000000000000000000000000000000 P e0dfa7f9a21b4c5aa45380d7fd8529fd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "445dc5aee8934a8aadf1de4114189469" member_type: VOTER last_known_addr { host: "127.2.2.62" port: 39037 } } peers { permanent_uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd" member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } } peers { permanent_uuid: "d76f973fad09443e80a402f2ad4e099d" member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } }
22:00:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.296697 2187 raft_consensus.cc:3037] T 00000000000000000000000000000000 P e0dfa7f9a21b4c5aa45380d7fd8529fd [term 0 FOLLOWER]: Advancing to term 1
22:00:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.300583 2188 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d76f973fad09443e80a402f2ad4e099d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd"
22:00:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.302204 2189 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d76f973fad09443e80a402f2ad4e099d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd" is_pre_election: true
22:00:33.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.307793 2187 raft_consensus.cc:1261] T 00000000000000000000000000000000 P e0dfa7f9a21b4c5aa45380d7fd8529fd [term 1 FOLLOWER]: Refusing update from remote peer d76f973fad09443e80a402f2ad4e099d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
22:00:33.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.309219 2215 sys_catalog.cc:564] T 00000000000000000000000000000000 P e0dfa7f9a21b4c5aa45380d7fd8529fd [sys.catalog]: configured and running, proceeding with master startup.
22:00:33.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.309882 2305 consensus_queue.cc:1035] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd" member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
22:00:33.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.313824 2345 sys_catalog.cc:455] T 00000000000000000000000000000000 P e0dfa7f9a21b4c5aa45380d7fd8529fd [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "445dc5aee8934a8aadf1de4114189469" member_type: VOTER last_known_addr { host: "127.2.2.62" port: 39037 } } peers { permanent_uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd" member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } } peers { permanent_uuid: "d76f973fad09443e80a402f2ad4e099d" member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } } }
22:00:33.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.314664 2345 sys_catalog.cc:458] T 00000000000000000000000000000000 P e0dfa7f9a21b4c5aa45380d7fd8529fd [sys.catalog]: This master's current role is: FOLLOWER
22:00:33.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.396898 2345 sys_catalog.cc:455] T 00000000000000000000000000000000 P e0dfa7f9a21b4c5aa45380d7fd8529fd [sys.catalog]: SysCatalogTable state changed. Reason: New leader d76f973fad09443e80a402f2ad4e099d. Latest consensus state: current_term: 1 leader_uuid: "d76f973fad09443e80a402f2ad4e099d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "445dc5aee8934a8aadf1de4114189469" member_type: VOTER last_known_addr { host: "127.2.2.62" port: 39037 } } peers { permanent_uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd" member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } } peers { permanent_uuid: "d76f973fad09443e80a402f2ad4e099d" member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } } }
22:00:33.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.397464 2345 sys_catalog.cc:458] T 00000000000000000000000000000000 P e0dfa7f9a21b4c5aa45380d7fd8529fd [sys.catalog]: This master's current role is: FOLLOWER
22:00:33.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.398847 2347 sys_catalog.cc:455] T 00000000000000000000000000000000 P e0dfa7f9a21b4c5aa45380d7fd8529fd [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d76f973fad09443e80a402f2ad4e099d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "445dc5aee8934a8aadf1de4114189469" member_type: VOTER last_known_addr { host: "127.2.2.62" port: 39037 } } peers { permanent_uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd" member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } } peers { permanent_uuid: "d76f973fad09443e80a402f2ad4e099d" member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } } }
22:00:33.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.399801 2347 sys_catalog.cc:458] T 00000000000000000000000000000000 P e0dfa7f9a21b4c5aa45380d7fd8529fd [sys.catalog]: This master's current role is: FOLLOWER
22:00:33.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.414618 2361 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
22:00:33.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:33.677443 2342 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:00:33.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:33.677978 2342 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:00:33.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:33.678449 2342 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:00:33.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:33.712203 2342 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:00:33.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:33.713243 2342 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.2.1
22:00:33.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.739796 2332 catalog_manager.cc:1259] Loaded cluster ID: 3cf0617eb5914ffcb5bd2cb55bae2918
22:00:33.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.740345 2332 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469: loading cluster ID for follower catalog manager: success
22:00:33.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.747691 2332 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469: acquiring CA information for follower catalog manager: success
22:00:33.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.753880 2332 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 445dc5aee8934a8aadf1de4114189469: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
22:00:33.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.771768 2342 tablet_server_runner.cc:78] Tablet server non-default flags:
22:00:33.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:00:33.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.2.20:39983
22:00:33.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:00:33.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:00:33.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-0/data
22:00:33.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-0/wal
22:00:33.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:00:33.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:00:33.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.2.1:0
22:00:33.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:00:33.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:00:33.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:00:33.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-0/data/info.pb
22:00:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.2.1
22:00:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:00:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.2.62:39037,127.2.2.61:43083,127.2.2.60:44059
22:00:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:00:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:00:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:00:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:00:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:00:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.2.1
22:00:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-0/logs
22:00:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:00:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:00:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
22:00:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:00:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
22:00:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:00:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
22:00:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
22:00:33.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:00:33.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.775079 2342 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:00:33.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.777673 2342 file_cache.cc:492] Constructed file cache file cache with capacity 419430
22:00:33.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:33.829957 2367 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:00:33.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.929543 2305 sys_catalog.cc:455] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d76f973fad09443e80a402f2ad4e099d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "445dc5aee8934a8aadf1de4114189469" member_type: VOTER last_known_addr { host: "127.2.2.62" port: 39037 } } peers { permanent_uuid: "e0dfa7f9a21b4c5aa45380d7fd8529fd" member_type: VOTER last_known_addr { host: "127.2.2.61" port: 43083 } } peers { permanent_uuid: "d76f973fad09443e80a402f2ad4e099d" member_type: VOTER last_known_addr { host: "127.2.2.60" port: 44059 } } }
22:00:33.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:33.930598 2305 sys_catalog.cc:458] T 00000000000000000000000000000000 P d76f973fad09443e80a402f2ad4e099d [sys.catalog]: This master's current role is: LEADER
22:00:34.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:34.439986 2361 catalog_manager.cc:1259] Loaded cluster ID: 3cf0617eb5914ffcb5bd2cb55bae2918
22:00:34.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:34.440320 2361 catalog_manager.cc:1552] T 00000000000000000000000000000000 P e0dfa7f9a21b4c5aa45380d7fd8529fd: loading cluster ID for follower catalog manager: success
22:00:34.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:34.457731 2361 catalog_manager.cc:1574] T 00000000000000000000000000000000 P e0dfa7f9a21b4c5aa45380d7fd8529fd: acquiring CA information for follower catalog manager: success
22:00:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:34.508900 2361 catalog_manager.cc:1602] T 00000000000000000000000000000000 P e0dfa7f9a21b4c5aa45380d7fd8529fd: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
22:00:35.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:35.208329 2366 debug-util.cc:398] Leaking SignalData structure 0x7b0800014fc0 after lost signal to thread 2342
22:00:35.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:35.524698 2342 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.691s user 0.264s sys 0.502s
22:00:35.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:35.525822 2342 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.692s user 0.264s sys 0.502s
22:00:35.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:35.525887 2369 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
22:00:35.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.526353 2342 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
22:00:35.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:33.836318 2368 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:00:35.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:35.526450 2371 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:00:35.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.531338 2342 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:00:35.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.534191 2342 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:00:35.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.535650 2342 hybrid_clock.cc:648] HybridClock initialized: now 1714687235535591 us; error 74 us; skew 500 ppm
22:00:35.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.536561 2342 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:00:35.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.544476 2342 webserver.cc:460] Webserver started at http://127.2.2.1:41947/ using document root <none> and password file <none>
22:00:35.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.545432 2342 fs_manager.cc:359] Metadata directory not provided
22:00:35.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.545667 2342 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:00:35.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.546217 2342 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:00:35.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.551659 2342 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-0/data/instance:
22:00:35.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff88b7355e6d4c3fa405b7a1da2486c7"
22:00:35.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 22:00:35 on dist-test-slave-ntzt"
22:00:35.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.553215 2342 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-0/wal/instance:
22:00:35.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff88b7355e6d4c3fa405b7a1da2486c7"
22:00:35.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 22:00:35 on dist-test-slave-ntzt"
22:00:35.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.561528 2342 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.002s
22:00:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.567970 2381 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:00:35.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.569212 2342 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
22:00:35.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.569574 2342 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-0/data,/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-0/wal
22:00:35.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff88b7355e6d4c3fa405b7a1da2486c7"
22:00:35.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 22:00:35 on dist-test-slave-ntzt"
22:00:35.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.569945 2342 fs_report.cc:383] FS layout report
22:00:35.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:00:35.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-0/wal
22:00:35.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-0/wal
22:00:35.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-0/data/data
22:00:35.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:00:35.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:00:35.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:00:35.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:00:35.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:00:35.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:00:35.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:00:35.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:00:35.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:00:35.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:00:35.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:00:35.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.633896 2342 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
22:00:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.635943 2342 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:00:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.636564 2342 kserver.cc:163] Server-wide thread pool size limit: 3276
22:00:35.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.640022 2342 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
22:00:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.646203 2342 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
22:00:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.646492 2342 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
22:00:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.646821 2342 ts_tablet_manager.cc:590] Registered 0 tablets
22:00:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.647073 2342 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
22:00:35.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.875085 2342 rpc_server.cc:307] RPC server started. Bound to: 127.2.2.1:46797
22:00:35.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.875212 2498 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.2.1:46797 every 8 connection(s)
22:00:35.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.878842 2342 server_base.cc:1166] Dumped server information to /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-0/data/info.pb
22:00:35.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.883145 2056 external_mini_cluster.cc:1388] Started /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu as pid 2342
22:00:35.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.883600 2056 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-0/wal/instance
22:00:35.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.899435 2056 external_mini_cluster.cc:1326] Running /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu
22:00:35.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu
22:00:35.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-1/wal
22:00:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-1/data
22:00:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:00:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:00:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:00:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:00:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:00:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:00:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-1/logs
22:00:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-1/data/info.pb
22:00:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:00:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:00:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:00:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:00:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:00:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:00:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:00:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:00:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
22:00:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:00:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.2.2:0
22:00:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.2.2
22:00:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.2.2
22:00:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:00:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.2.62:39037,127.2.2.61:43083,127.2.2.60:44059
22:00:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.2.20:39983
22:00:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:00:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:00:35.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
22:00:35.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.982618 2499 heartbeater.cc:347] Connected to a master server at 127.2.2.60:44059
22:00:35.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.983115 2499 heartbeater.cc:464] Registering TS with master...
22:00:35.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.984441 2499 heartbeater.cc:511] Master 127.2.2.60:44059 requested a full tablet report, sending...
22:00:35.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.987955 2260 ts_manager.cc:194] Registered new tserver with Master: ff88b7355e6d4c3fa405b7a1da2486c7 (127.2.2.1:46797)
22:00:35.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.990680 2260 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.2.2.1:55503
22:00:36.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.999054 2500 heartbeater.cc:347] Connected to a master server at 127.2.2.61:43083
22:00:36.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.999425 2500 heartbeater.cc:464] Registering TS with master...
22:00:36.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:35.999873 2504 heartbeater.cc:347] Connected to a master server at 127.2.2.62:39037
22:00:36.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:36.000109 2500 heartbeater.cc:511] Master 127.2.2.61:43083 requested a full tablet report, sending...
22:00:36.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:36.000241 2504 heartbeater.cc:464] Registering TS with master...
22:00:36.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:36.001067 2504 heartbeater.cc:511] Master 127.2.2.62:39037 requested a full tablet report, sending...
22:00:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:36.004951 2102 ts_manager.cc:194] Registered new tserver with Master: ff88b7355e6d4c3fa405b7a1da2486c7 (127.2.2.1:46797)
22:00:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:36.005306 2179 ts_manager.cc:194] Registered new tserver with Master: ff88b7355e6d4c3fa405b7a1da2486c7 (127.2.2.1:46797)
22:00:36.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:36.382012 2503 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:00:36.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:36.382571 2503 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:00:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:36.383126 2503 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:00:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:36.416687 2503 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:00:36.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:36.417973 2503 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.2.2
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:36.453809 2503 tablet_server_runner.cc:78] Tablet server non-default flags:
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.2.20:39983
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-1/data
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-1/wal
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.2.2:0
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-1/data/info.pb
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.2.2
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.2.62:39037,127.2.2.61:43083,127.2.2.60:44059
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.2.2
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-1/logs
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:00:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:36.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
22:00:36.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:00:36.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
22:00:36.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:00:36.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
22:00:36.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
22:00:36.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:00:36.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:36.455334 2503 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:00:36.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:36.457042 2503 file_cache.cc:492] Constructed file cache file cache with capacity 419430
22:00:36.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:36.479516 2515 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:00:36.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:36.995962 2499 heartbeater.cc:503] Master 127.2.2.60:44059 was elected leader, sending a full tablet report...
22:00:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:37.874939 2514 debug-util.cc:398] Leaking SignalData structure 0x7b0800014fc0 after lost signal to thread 2503
22:00:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:37.940028 2514 kernel_stack_watchdog.cc:198] Thread 2503 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:637 for 390ms:
22:00:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
22:00:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
22:00:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
22:00:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
22:00:37.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:36.480366 2516 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:00:37.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:37.946336 2517 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1459 milliseconds
22:00:37.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:37.947026 2503 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.464s user 0.006s sys 0.000s
22:00:37.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:37.947289 2503 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.465s user 0.006s sys 0.000s
22:00:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:37.948757 2519 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:00:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:37.948820 2503 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
22:00:37.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:37.949944 2503 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:00:37.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:37.952489 2503 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:00:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:37.953899 2503 hybrid_clock.cc:648] HybridClock initialized: now 1714687237953869 us; error 54 us; skew 500 ppm
22:00:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:37.954711 2503 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:00:37.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:37.962164 2503 webserver.cc:460] Webserver started at http://127.2.2.2:42213/ using document root <none> and password file <none>
22:00:37.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:37.963097 2503 fs_manager.cc:359] Metadata directory not provided
22:00:37.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:37.963322 2503 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:00:37.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:37.963788 2503 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:00:37.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:37.969252 2503 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-1/data/instance:
22:00:37.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d3387e6e9106431b9822bd9457cad32d"
22:00:37.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 22:00:37 on dist-test-slave-ntzt"
22:00:37.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:37.970393 2503 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-1/wal/instance:
22:00:37.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d3387e6e9106431b9822bd9457cad32d"
22:00:37.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 22:00:37 on dist-test-slave-ntzt"
22:00:37.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:37.978368 2503 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.003s sys 0.004s
22:00:37.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:37.984247 2527 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:00:37.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:37.985469 2503 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
22:00:37.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:37.985806 2503 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-1/data,/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-1/wal
22:00:37.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d3387e6e9106431b9822bd9457cad32d"
22:00:37.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 22:00:37 on dist-test-slave-ntzt"
22:00:37.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:37.986157 2503 fs_report.cc:383] FS layout report
22:00:37.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:00:37.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-1/wal
22:00:37.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-1/wal
22:00:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-1/data/data
22:00:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:00:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:00:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:00:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:00:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:00:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:00:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:00:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:00:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:00:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:00:37.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:00:38.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.046947 2503 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
22:00:38.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.048600 2503 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:00:38.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.049099 2503 kserver.cc:163] Server-wide thread pool size limit: 3276
22:00:38.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.052541 2503 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
22:00:38.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.057655 2503 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
22:00:38.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.057883 2503 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
22:00:38.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.058198 2503 ts_tablet_manager.cc:590] Registered 0 tablets
22:00:38.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.058411 2503 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
22:00:38.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.238308 2503 rpc_server.cc:307] RPC server started. Bound to: 127.2.2.2:38243
22:00:38.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.238420 2643 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.2.2:38243 every 8 connection(s)
22:00:38.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.240809 2503 server_base.cc:1166] Dumped server information to /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-1/data/info.pb
22:00:38.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.248435 2056 external_mini_cluster.cc:1388] Started /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu as pid 2503
22:00:38.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.249281 2056 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-1/wal/instance
22:00:38.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.267311 2056 external_mini_cluster.cc:1326] Running /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-2/wal
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-2/data
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-2/logs
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-2/data/info.pb
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.2.3:0
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.2.3
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.2.3
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.2.62:39037,127.2.2.61:43083,127.2.2.60:44059
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.2.20:39983
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:00:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
22:00:38.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.279492 2645 heartbeater.cc:347] Connected to a master server at 127.2.2.61:43083
22:00:38.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.280050 2645 heartbeater.cc:464] Registering TS with master...
22:00:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.281761 2645 heartbeater.cc:511] Master 127.2.2.61:43083 requested a full tablet report, sending...
22:00:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.282058 2644 heartbeater.cc:347] Connected to a master server at 127.2.2.60:44059
22:00:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.282428 2644 heartbeater.cc:464] Registering TS with master...
22:00:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.283453 2644 heartbeater.cc:511] Master 127.2.2.60:44059 requested a full tablet report, sending...
22:00:38.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.283800 2646 heartbeater.cc:347] Connected to a master server at 127.2.2.62:39037
22:00:38.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.284116 2646 heartbeater.cc:464] Registering TS with master...
22:00:38.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.285140 2646 heartbeater.cc:511] Master 127.2.2.62:39037 requested a full tablet report, sending...
22:00:38.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.286407 2179 ts_manager.cc:194] Registered new tserver with Master: d3387e6e9106431b9822bd9457cad32d (127.2.2.2:38243)
22:00:38.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.286664 2260 ts_manager.cc:194] Registered new tserver with Master: d3387e6e9106431b9822bd9457cad32d (127.2.2.2:38243)
22:00:38.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.288365 2102 ts_manager.cc:194] Registered new tserver with Master: d3387e6e9106431b9822bd9457cad32d (127.2.2.2:38243)
22:00:38.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.288566 2260 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.2.2.2:45083
22:00:38.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:38.587558 2654 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:00:38.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:38.588058 2654 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:00:38.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:38.588554 2654 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:00:38.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:38.620633 2654 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:00:38.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:38.621456 2654 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.2.3
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.655813 2654 tablet_server_runner.cc:78] Tablet server non-default flags:
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.2.20:39983
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-2/data
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-2/wal
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.2.3:0
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-2/data/info.pb
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.2.3
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.2.62:39037,127.2.2.61:43083,127.2.2.60:44059
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.2.3
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-2/logs
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
22:00:38.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:00:38.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.657027 2654 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:00:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.658578 2654 file_cache.cc:492] Constructed file cache file cache with capacity 419430
22:00:38.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:38.675865 2660 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:00:39.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:39.293161 2644 heartbeater.cc:503] Master 127.2.2.60:44059 was elected leader, sending a full tablet report...
22:00:39.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==2654==WARNING: Can't read from symbolizer at fd 5
22:00:39.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==2654==WARNING: invalid path to external symbolizer!
22:00:39.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==2654==WARNING: Can't write to symbolizer at fd 8
22:00:39.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==2654==WARNING: Can't write to symbolizer at fd 8
22:00:39.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==2654==WARNING: Can't write to symbolizer at fd 8
22:00:39.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==2654==WARNING: Failed to use and restart external symbolizer!
22:00:39.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
22:00:39.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=2654)
22:00:39.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 8 at 0x7b0400006940 by thread T6:
22:00:39.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x47da7f)
22:00:39.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libc.so.6+0xd3fe4)
22:00:39.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libglog.so.1+0x1f60b)
22:00:39.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libglog.so.1+0x17b16)
22:00:39.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libglog.so.1+0x18f10)
22:00:39.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libkudu_cloud_util.so+0x11a61)
22:00:39.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libkudu_cloud_util.so+0x12d7d)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libkudu_cloud_util.so+0x12d09)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libkudu_cloud_util.so+0x12c99)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (libkudu_cloud_util.so+0x12c61)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (libkudu_cloud_util.so+0x11f5d)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x589804)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x589639)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_util.so+0x437936)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x44ffbf)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous write of size 8 at 0x7b0400006940 by thread T5:
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x47a99c)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libc.so.6+0x9daf9)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libglog.so.1+0x1f60b)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libglog.so.1+0x17b16)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libglog.so.1+0x18f10)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libkudu_cloud_util.so+0x11a61)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libkudu_cloud_util.so+0x12d7d)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libkudu_cloud_util.so+0x12d09)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libkudu_cloud_util.so+0x12c99)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (libkudu_cloud_util.so+0x12c61)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (libkudu_cloud_util.so+0x11f5d)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x589804)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x589639)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_util.so+0x437936)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x44ffbf)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T6 'Azure-2661' (tid=2661, running) created by main thread at:
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x453a75)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libkudu_util.so+0x43719a)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (kudu+0x579859)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libkudu_cloud_util.so+0x114e5)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libserver_process.so+0x143586)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libserver_process.so+0x1419e6)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libkserver.so+0x13191)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libtserver.so+0x255912)
22:00:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libtserver.so+0x263211)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x608029)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x511021)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x510f8c)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x510efc)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x50fb65)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x127dcf)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (libkudu_tools_util.so+0x11a631)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (libkudu_tools_util.so+0x118362)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x623da0)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (kudu+0x6246d7)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (kudu+0x624c1e)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (libc.so.6+0x21bf6)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T5 'AWS-2660' (tid=2660, finished) created by main thread at:
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x453a75)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libkudu_util.so+0x43719a)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (kudu+0x579859)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libkudu_cloud_util.so+0x114e5)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libserver_process.so+0x143586)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libserver_process.so+0x1419e6)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libkserver.so+0x13191)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libtserver.so+0x255912)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libtserver.so+0x263211)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x608029)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x511021)
22:00:39.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x510f8c)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x510efc)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x50fb65)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x127dcf)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (libkudu_tools_util.so+0x11a631)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (libkudu_tools_util.so+0x118362)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x623da0)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (kudu+0x6246d7)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (kudu+0x624c1e)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (libc.so.6+0x21bf6)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-task8UTc2V/build/tsan/bin/kudu+0x47da7f)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=2654)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Read of size 8 at 0x7f560e4f9700 by thread T7:
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libkudu_util.so+0x437b45)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libkudu_util.so+0x441dfc)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libkudu_util.so+0x437952)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (kudu+0x44ffbf)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous write of size 8 at 0x7f560e4f9700 by thread T5 (mutexes: write M83):
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libglog.so.1+0x2b07e)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libkudu_util.so+0x437b80)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libkudu_util.so+0x441dfc)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libkudu_util.so+0x437952)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (kudu+0x44ffbf)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Location is global '<null>' at 0x000000000000 (libkudu_util.so+0x0000004df700)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mutex M83 (0x7f560c44f840) created at:
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x48425b)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libglog.so.1+0xe6ec)
22:00:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (ld-linux-x86-64.so.2+0x108f2)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T7 'GCE-2662'W20240502 22:00:38.676534 2663 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (tid=2662, running) created by main thread at:
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x453a75)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libkudu_util.so+0x43719a)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (kudu+0x579859)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libkudu_cloud_util.so+0x114e5)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libserver_process.so+0x143586)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libserver_process.so+0x1419e6)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libkserver.so+0x13191)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libtserver.so+0x255912)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libtserver.so+0x263211)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x608029)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x511021)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x510f8c)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x510efc)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x50fb65)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x127dcf)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (libkudu_tools_util.so+0x11a631)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (libkudu_tools_util.so+0x118362)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x623da0)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (kudu+0x6246d7)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (kudu+0x624c1e)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (libc.so.6+0x21bf6)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T5 'AWS-2660' (tid=2660, finished) created by main thread at:
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x453a75)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libkudu_util.so+0x43719a)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (kudu+0x579859)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:38.677480 2654 server_base.cc:1034] running on GCE node
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libkudu_cloud_util.so+0x114e5)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libserver_process.so+0x143586)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libserver_process.so+0x1419e6)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libkserver.so+0x13191)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libtserver.so+0x255912)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libtserver.so+0x263211)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x608029)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x511021)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x510f8c)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x510efc)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x50fb65)
22:00:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x127dcf)
22:00:39.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (libkudu_tools_util.so+0x11a631)
22:00:39.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (libkudu_tools_util.so+0x118362)
22:00:39.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x623da0)
22:00:39.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (kudu+0x6246d7)
22:00:39.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (kudu+0x624c1e)
22:00:39.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (libc.so.6+0x21bf6)
22:00:39.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:39.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:38.676398 2661 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:00:39.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-task8UTc2V/build/tsan/lib/libkudu_util.so+0x437b45)
22:00:39.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
22:00:39.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:39.635584 2654 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:00:39.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:39.638772 2654 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:00:39.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:39.640255 2654 hybrid_clock.cc:648] HybridClock initialized: now 1714687239640210 us; error 34 us; skew 500 ppm
22:00:39.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:39.641356 2654 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:00:39.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:39.649510 2654 webserver.cc:460] Webserver started at http://127.2.2.3:38167/ using document root <none> and password file <none>
22:00:39.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:39.651109 2654 fs_manager.cc:359] Metadata directory not provided
22:00:39.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:39.651444 2654 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:00:39.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:39.651791 2654 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:00:39.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:39.652647 2654 fs_manager.cc:359] Metadata directory not provided
22:00:39.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:39.652822 2654 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:00:39.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:39.653525 2654 ts_tablet_manager.cc:1470] Shutting down tablet manager...
22:00:39.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
22:00:39.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=2654)
22:00:39.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 4 at 0x7b6800003600 by main thread (mutexes: write M8016985954727800):
22:00:39.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1bb6d4)
22:00:39.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x18c7f2)
22:00:39.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x141322)
22:00:39.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libserver_process.so+0x1410b6)
22:00:39.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmaster.so+0x4019ff)
22:00:39.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libtserver.so+0x2552b2)
22:00:39.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libtserver.so+0x26329a)
22:00:39.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x608029)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x511021)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x510f8c)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x510efc)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x50fb65)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x127dcf)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x11a631)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x118362)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x623da0)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x6246d7)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x624c1e)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libc.so.6+0x21bf6)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous read of size 4 at 0x7b6800003600 by thread T12:
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1bd0bf)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (kudu+0x44ffbf)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Location is heap block of size 1368 at 0x7b6800003600 allocated by main thread:
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x4801dc)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1bbbe4)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18d750)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libserver_process.so+0x141ede)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libkserver.so+0x13191)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libtserver.so+0x255912)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libtserver.so+0x263211)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x608029)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x511021)
22:00:39.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x510f8c)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x510efc)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x50fb65)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x127dcf)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x11a631)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x118362)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x623da0)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x6246d7)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x624c1e)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libc.so.6+0x21bf6)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mutex M8016985954727800 is already destroyed.
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T12 'sq_acceptor' (tid=2670, running) created by main thread at:
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x453a75)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1bce97)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18d750)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libserver_process.so+0x141ede)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libkserver.so+0x13191)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libtserver.so+0x255912)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libtserver.so+0x263211)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x608029)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x511021)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x510f8c)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x510efc)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x50fb65)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x127dcf)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x11a631)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x118362)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x623da0)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x6246d7)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x624c1e)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libc.so.6+0x21bf6)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-task8UTc2V/build/tsan/lib/libserver_process.so+0x1bb6d4)
22:00:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=2654)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 8 at 0x7b0c0003ed90 by thread T12:
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x47da7f)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1bd150)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (kudu+0x44ffbf)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous read of size 4 at 0x7b0c0003ed90 by main thread:
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1b6508)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x18e30b)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18d83b)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libserver_process.so+0x141ede)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libkserver.so+0x13191)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libtserver.so+0x255912)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libtserver.so+0x263211)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x608029)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x511021)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x510f8c)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x510efc)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x50fb65)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x127dcf)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x11a631)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x118362)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x623da0)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x6246d7)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x624c1e)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libc.so.6+0x21bf6)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T12 'sq_acceptor' (tid=2670, running) created by main thread at:
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x453a75)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1bce97)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18d750)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libserver_process.so+0x141ede)
22:00:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libkserver.so+0x13191)
22:00:39.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libtserver.so+0x255912)
22:00:39.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libtserver.so+0x263211)
22:00:39.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x608029)
22:00:39.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x511021)
22:00:39.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x510f8c)
22:00:39.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x510efc)
22:00:39.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x50fb65)
22:00:39.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x127dcf)
22:00:39.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x11a631)
22:00:39.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x118362)
22:00:39.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x623da0)
22:00:39.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x6246d7)
22:00:39.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x624c1e)
22:00:39.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libc.so.6+0x21bf6)
22:00:39.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:39.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-task8UTc2V/build/tsan/bin/kudu+0x47da7f)
22:00:39.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
22:00:39.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Could not create new FS layout: Write-ahead log directory /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-2/wal failed to canonicalize: Unable to canonicalize /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301/ts-2: No such file or directory (error 2)
22:00:40.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ThreadSanitizer: reported 4 warnings
22:00:40.727 [DEBUG - main] (MiniKuduCluster.java:195) Response: error {
code: RUNTIME_ERROR
message: "failed to start tablet server 3: /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu: process exited with non-zero status 66"
}
22:00:40.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:40.727895 2056 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu with pid 2342
22:00:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:40.768630 2056 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu with pid 2503
22:00:40.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:40.815954 2056 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu with pid 2069
22:00:40.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:40.864671 2056 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu with pid 2141
22:00:40.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 22:00:40.901026 2056 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu with pid 2217
22:00:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:40.957347 2056 env_posix.cc:2125] Unable to access file /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301 during walk: No such file or directory
22:00:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:40.957630 2056 env_posix.cc:2127] Unable to access file /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301 during walk: No such file or directory
22:00:40.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 22:00:40.957975 2056 tool_action_test.cc:443] Could not delete cluster root: IO error: /tmp/dist-test-task8UTc2V/test-tmp/mini-kudu-cluster3877258909995246301: One or more errors occurred
22:00:40.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==2056==WARNING: invalid path to external symbolizer!
22:00:40.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==2056==WARNING: Failed to use and restart external symbolizer!
22:00:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
22:00:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=2056)
22:00:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 4 at 0x7b6800003c00 by main thread (mutexes: write M3825):
22:00:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1bb6d4)
22:00:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x18c7f2)
22:00:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libmini_oidc.so+0x21aa2)
22:00:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x219ea)
22:00:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0xb251e)
22:00:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libmini_cluster.so+0x97bfd)
22:00:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libmini_cluster.so+0x96b9b)
22:00:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libmini_cluster.so+0x835ca)
22:00:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libmini_cluster.so+0x83729)
22:00:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x635547)
22:00:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x6319bd)
22:00:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x63108b)
22:00:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x62e2c0)
22:00:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x511021)
22:00:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x510f8c)
22:00:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x510efc)
22:00:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x50fb65)
22:00:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libkudu_tools_util.so+0x127dcf)
22:00:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libkudu_tools_util.so+0x11a631)
22:00:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (libkudu_tools_util.so+0x118362)
22:00:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (kudu+0x623da0)
22:00:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #21 <null> <null> (kudu+0x6246d7)
22:00:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #22 <null> <null> (kudu+0x624c1e)
22:00:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #23 <null> <null> (libc.so.6+0x21bf6)
22:00:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous read of size 4 at 0x7b6800003c00 by thread T5:
22:00:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1bd347)
22:00:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (kudu+0x44ffbf)
22:00:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) As if synchronized via sleep:
22:00:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x474fa2)
22:00:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libgutil.so+0xb5f1a)
22:00:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libkudu_util.so+0x3cd336)
22:00:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_cluster.so+0x8f5ae)
22:00:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x92a87)
22:00:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libmini_cluster.so+0x8796f)
22:00:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libmini_cluster.so+0x85baa)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x62f8a9)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x62e1a4)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x511021)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x510f8c)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x510efc)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x50fb65)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x127dcf)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x11a631)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (libkudu_tools_util.so+0x118362)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x623da0)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x6246d7)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (kudu+0x624c1e)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (libc.so.6+0x21bf6)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Location is heap block of size 1368 at 0x7b6800003c00 allocated by main thread:
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x4801dc)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1bbbe4)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18d750)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x2218c)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x846e4)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x62f8a9)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x62e1a4)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x511021)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x510f8c)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x510efc)
22:00:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x50fb65)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x127dcf)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x11a631)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x118362)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x623da0)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6246d7)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x624c1e)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mutex M3825 (0x7b6800003d78) created at:
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x453fe2)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1bc961)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18d750)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x2218c)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x846e4)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x62f8a9)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x62e1a4)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x511021)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x510f8c)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x510efc)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x50fb65)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x127dcf)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x11a631)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x118362)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x623da0)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6246d7)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x624c1e)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T5 'sq_acceptor' (tid=2064, running) created by main thread at:
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x453a75)
22:00:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1bce97)
22:00:40.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18d750)
22:00:40.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x2218c)
22:00:40.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x846e4)
22:00:40.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x62f8a9)
22:00:40.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x62e1a4)
22:00:40.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x511021)
22:00:40.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x510f8c)
22:00:40.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x510efc)
22:00:40.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x50fb65)
22:00:40.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x127dcf)
22:00:40.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x11a631)
22:00:40.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x118362)
22:00:40.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x623da0)
22:00:40.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6246d7)
22:00:40.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x624c1e)
22:00:40.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
22:00:40.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:40.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-task8UTc2V/build/tsan/lib/libserver_process.so+0x1bb6d4)
22:00:40.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
22:00:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
22:00:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=2056)
22:00:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 4 at 0x7b6800003600 by main thread (mutexes: write M3822):
22:00:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1bb6d4)
22:00:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x18c7f2)
22:00:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libmini_oidc.so+0x21ac5)
22:00:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x219ea)
22:00:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0xb251e)
22:00:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libmini_cluster.so+0x97bfd)
22:00:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libmini_cluster.so+0x96b9b)
22:00:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libmini_cluster.so+0x835ca)
22:00:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libmini_cluster.so+0x83729)
22:00:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x635547)
22:00:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x6319bd)
22:00:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x63108b)
22:00:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x62e2c0)
22:00:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x511021)
22:00:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x510f8c)
22:00:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x510efc)
22:00:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x50fb65)
22:00:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libkudu_tools_util.so+0x127dcf)
22:00:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libkudu_tools_util.so+0x11a631)
22:00:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (libkudu_tools_util.so+0x118362)
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (kudu+0x623da0)
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #21 <null> <null> (kudu+0x6246d7)
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #22 <null> <null> (kudu+0x624c1e)
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #23 <null> <null> (libc.so.6+0x21bf6)
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous read of size 4 at 0x7b6800003600 by thread T4:
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1bd347)
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (kudu+0x44ffbf)
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) As if synchronized via sleep:
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x4749dd)
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1bb779)
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18c7f2)
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x21aa2)
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_oidc.so+0x219ea)
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libmini_cluster.so+0xb251e)
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libmini_cluster.so+0x97bfd)
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libmini_cluster.so+0x96b9b)
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libmini_cluster.so+0x835ca)
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (libmini_cluster.so+0x83729)
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x635547)
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x6319bd)
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x63108b)
22:00:40.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x62e2c0)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x511021)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x510f8c)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x510efc)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x50fb65)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libkudu_tools_util.so+0x127dcf)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (libkudu_tools_util.so+0x11a631)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (libkudu_tools_util.so+0x118362)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #21 <null> <null> (kudu+0x623da0)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #22 <null> <null> (kudu+0x6246d7)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #23 <null> <null> (kudu+0x624c1e)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #24 <null> <null> (libc.so.6+0x21bf6)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Location is heap block of size 1368 at 0x7b6800003600 allocated by main thread:
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x4801dc)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1bbbe4)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18d750)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x21dd6)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x846e4)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x62f8a9)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x62e1a4)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x511021)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x510f8c)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x510efc)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x50fb65)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x127dcf)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x11a631)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x118362)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x623da0)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6246d7)
22:00:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x624c1e)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mutex M3822 (0x7b6800003778) created at:
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x453fe2)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1bc961)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18d750)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x21dd6)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x846e4)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x62f8a9)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x62e1a4)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x511021)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x510f8c)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x510efc)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x50fb65)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x127dcf)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x11a631)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x118362)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x623da0)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6246d7)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x624c1e)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T4 'sq_acceptor' (tid=2063, running) created by main thread at:
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x453a75)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1bce97)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18d750)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x21dd6)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x846e4)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x62f8a9)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x62e1a4)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x511021)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x510f8c)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x510efc)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x50fb65)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x127dcf)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x11a631)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x118362)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x623da0)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6246d7)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x624c1e)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-task8UTc2V/build/tsan/lib/libserver_process.so+0x1bb6d4)
22:00:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
22:00:40.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T22:00:40Z chronyd exiting
22:00:42.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ThreadSanitizer: reported 2 warnings
22:00:42.045 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduClient.testScanWithPredicates: failed attempt 1
java.io.IOException: failed to start tablet server 3: /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu: process exited with non-zero status 66
at org.apache.kudu.test.cluster.MiniKuduCluster.sendRequestToCluster(MiniKuduCluster.java:199) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster.start(MiniKuduCluster.java:266) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster.access$300(MiniKuduCluster.java:85) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster$MiniKuduClusterBuilder.build(MiniKuduCluster.java:883) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:156) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.io.IOException: failed to start tablet server 3: /tmp/dist-test-task8UTc2V/build/tsan/bin/kudu: process exited with non-zero status 66
at org.apache.kudu.test.cluster.MiniKuduCluster.sendRequestToCluster(MiniKuduCluster.java:199)
at org.apache.kudu.test.cluster.MiniKuduCluster.start(MiniKuduCluster.java:266)
at org.apache.kudu.test.cluster.MiniKuduCluster.access$300(MiniKuduCluster.java:85)
at org.apache.kudu.test.cluster.MiniKuduCluster$MiniKuduClusterBuilder.build(MiniKuduCluster.java:883)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:156)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)