15:56:45.020 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
15:56:45.020 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskhTbTJZ/test-tmp
15:56:45.020 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:56:45.020 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskhTbTJZ/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:56:45.022 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
15:56:45.355 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:56:45.355 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:56:45.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:45.358361 30486 mini_oidc.cc:150] Starting JWKS server
15:56:45.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:45.361539 30486 webserver.cc:458] Webserver started at http://127.0.0.1:35269/ using document root <none> and password file <none>
15:56:45.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:45.361919 30486 mini_oidc.cc:186] Starting OIDC Discovery server
15:56:45.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:45.363533 30486 webserver.cc:458] Webserver started at http://127.0.0.1:33229/ using document root <none> and password file <none>
15:56:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-14T15:56:45Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
15:56:45.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-14T15:56:45Z Disabled control of system clock
15:56:45.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:45.425387 30486 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu
15:56:45.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu
15:56:45.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-0/wal
15:56:45.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-0/data
15:56:45.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:56:45.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:56:45.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:56:45.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:56:45.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:56:45.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:56:45.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-0/logs
15:56:45.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-0/data/info.pb
15:56:45.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:56:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:56:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:56:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:56:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:56:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:56:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:56:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:56:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:56:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:56:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:56:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:56:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.197.190:35785
15:56:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.197.190
15:56:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:56:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.197.148:40779
15:56:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:56:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:56:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:56:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:56:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.197.190:35785,127.29.197.189:46717,127.29.197.188:41343
15:56:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:56:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35269/jwks.json with env {}
15:56:45.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:45.720041 30498 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:56:45.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:45.720614 30498 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:56:45.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:45.721089 30498 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:56:45.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:45.752553 30498 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:56:45.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:45.752835 30498 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:56:45.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:45.753033 30498 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:56:45.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:45.753226 30498 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:56:45.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:45.753418 30498 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:56:45.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:45.753623 30498 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35269/jwks.json
15:56:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:45.788312 30498 master_runner.cc:386] Master server non-default flags:
15:56:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:56:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.197.148:40779
15:56:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:56:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:56:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-0/data
15:56:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-0/wal
15:56:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:56:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.197.190:35785,127.29.197.189:46717,127.29.197.188:41343
15:56:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:56:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:56:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:56:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.197.190:35785
15:56:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35269/jwks.json
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-0/data/info.pb
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.197.190
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-0/logs
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 14:43:21 UTC on a633016ff5c9
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4176
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:56:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:45.789652 30498 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:56:45.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:45.791113 30498 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:56:45.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:45.807005 30504 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:56:46.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:45.807691 30507 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:56:46.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:45.808303 30505 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:56:46.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:45.809240 30498 server_base.cc:1034] running on GCE node
15:56:46.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:46.983605 30498 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:56:46.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:46.986374 30498 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:56:46.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:46.987740 30498 hybrid_clock.cc:648] HybridClock initialized: now 1736870206987697 us; error 53 us; skew 500 ppm
15:56:46.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:46.988567 30498 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:56:46.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:46.994689 30498 webserver.cc:458] Webserver started at http://127.29.197.190:33063/ using document root <none> and password file <none>
15:56:46.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:46.995625 30498 fs_manager.cc:362] Metadata directory not provided
15:56:46.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:46.995846 30498 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:56:46.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:46.996301 30498 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:56:47.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.001488 30498 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-0/data/instance:
15:56:47.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "26206da836ae45d898f608eb3d6e5587"
15:56:47.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 15:56:46 on dist-test-slave-pjn3"
15:56:47.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.002746 30498 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-0/wal/instance:
15:56:47.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "26206da836ae45d898f608eb3d6e5587"
15:56:47.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 15:56:46 on dist-test-slave-pjn3"
15:56:47.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.009811 30498 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.003s sys 0.005s
15:56:47.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.015046 30515 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:56:47.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.016005 30498 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
15:56:47.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.016330 30498 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-0/data,/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-0/wal
15:56:47.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "26206da836ae45d898f608eb3d6e5587"
15:56:47.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 15:56:46 on dist-test-slave-pjn3"
15:56:47.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.016680 30498 fs_report.cc:389] FS layout report
15:56:47.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:56:47.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-0/wal
15:56:47.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-0/wal
15:56:47.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-0/data/data
15:56:47.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:56:47.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:56:47.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:56:47.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:56:47.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:56:47.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:56:47.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:56:47.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:56:47.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:56:47.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:56:47.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:56:47.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.073660 30498 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:56:47.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.075325 30498 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:56:47.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.075865 30498 kserver.cc:163] Server-wide thread pool size limit: 3276
15:56:47.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.146031 30498 rpc_server.cc:307] RPC server started. Bound to: 127.29.197.190:35785
15:56:47.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.146116 30566 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.197.190:35785 every 8 connection(s)
15:56:47.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.148643 30498 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-0/data/info.pb
15:56:47.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.152935 30486 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu as pid 30498
15:56:47.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.153475 30486 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-0/wal/instance
15:56:47.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.154667 30567 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.158888 30486 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-1/wal
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-1/data
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-1/logs
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-1/data/info.pb
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.197.189:46717
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.197.189
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.197.148:40779
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.197.190:35785,127.29.197.189:46717,127.29.197.188:41343
15:56:47.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:56:47.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35269/jwks.json with env {}
15:56:47.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.169329 30567 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.197.190" port: 35785 } has no permanent_uuid. Determining permanent_uuid...
15:56:47.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.192787 30567 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } has no permanent_uuid. Determining permanent_uuid...
15:56:47.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:47.194954 30517 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.197.189:46717: connect: Connection refused (error 111)
15:56:47.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:47.198987 30567 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.197.189:46717: Network error: Client connection negotiation failed: client connection to 127.29.197.189:46717: connect: Connection refused (error 111)
15:56:47.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.248432 30567 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } attempt: 1
15:56:47.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:47.252282 30567 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.197.189:46717: Network error: Client connection negotiation failed: client connection to 127.29.197.189:46717: connect: Connection refused (error 111)
15:56:47.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.320813 30567 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } attempt: 2
15:56:47.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:47.324764 30567 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.197.189:46717: Network error: Client connection negotiation failed: client connection to 127.29.197.189:46717: connect: Connection refused (error 111)
15:56:47.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.416226 30567 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } attempt: 3
15:56:47.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:47.420037 30567 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.197.189:46717: Network error: Client connection negotiation failed: client connection to 127.29.197.189:46717: connect: Connection refused (error 111)
15:56:47.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:47.459760 30569 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:56:47.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:47.460305 30569 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:56:47.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:47.460778 30569 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:56:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:47.490983 30569 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:56:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:47.491253 30569 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:56:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:47.491454 30569 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:56:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:47.491654 30569 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:56:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:47.491840 30569 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:56:47.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:47.491982 30569 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35269/jwks.json
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.526922 30569 master_runner.cc:386] Master server non-default flags:
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.197.148:40779
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-1/data
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-1/wal
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.197.190:35785,127.29.197.189:46717,127.29.197.188:41343
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.197.189:46717
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35269/jwks.json
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-1/data/info.pb
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.197.189
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-1/logs
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:56:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:56:47.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:56:47.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
15:56:47.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:56:47.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 14:43:21 UTC on a633016ff5c9
15:56:47.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4176
15:56:47.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:56:47.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.528187 30569 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:56:47.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.529702 30569 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:56:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:47.544157 30578 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:56:47.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.563481 30567 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } attempt: 4
15:56:47.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:47.567245 30567 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.197.189:46717: Network error: Client connection negotiation failed: client connection to 127.29.197.189:46717: connect: Connection refused (error 111)
15:56:47.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.866891 30567 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } attempt: 5
15:56:47.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:47.871059 30567 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.197.189:46717: Network error: Client connection negotiation failed: client connection to 127.29.197.189:46717: connect: Connection refused (error 111)
15:56:48.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.418637 30567 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } attempt: 6
15:56:48.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:48.426144 30567 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.197.189:46717: Network error: Client connection negotiation failed: client connection to 127.29.197.189:46717: connect: Connection refused (error 111)
15:56:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:47.544611 30579 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:56:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:47.545357 30581 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:56:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:47.546634 30569 server_base.cc:1034] running on GCE node
15:56:48.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.732241 30569 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:56:48.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.735250 30569 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:56:48.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.736672 30569 hybrid_clock.cc:648] HybridClock initialized: now 1736870208736633 us; error 43 us; skew 500 ppm
15:56:48.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.737447 30569 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:56:48.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.744130 30569 webserver.cc:458] Webserver started at http://127.29.197.189:43085/ using document root <none> and password file <none>
15:56:48.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.745014 30569 fs_manager.cc:362] Metadata directory not provided
15:56:48.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.745203 30569 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:56:48.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.745638 30569 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:56:48.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.750452 30569 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-1/data/instance:
15:56:48.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "498e4c9ae2b54c1b917287f14f0e1b47"
15:56:48.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 15:56:48 on dist-test-slave-pjn3"
15:56:48.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.751489 30569 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-1/wal/instance:
15:56:48.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "498e4c9ae2b54c1b917287f14f0e1b47"
15:56:48.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 15:56:48 on dist-test-slave-pjn3"
15:56:48.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.758412 30569 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
15:56:48.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.763567 30590 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:56:48.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.764501 30569 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
15:56:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.764783 30569 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-1/data,/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-1/wal
15:56:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "498e4c9ae2b54c1b917287f14f0e1b47"
15:56:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 15:56:48 on dist-test-slave-pjn3"
15:56:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.765071 30569 fs_report.cc:389] FS layout report
15:56:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:56:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-1/wal
15:56:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-1/wal
15:56:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-1/data/data
15:56:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:56:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:56:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:56:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:56:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:56:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:56:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:56:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:56:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:56:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:56:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:56:48.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.820381 30569 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:56:48.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.821779 30569 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:56:48.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.822166 30569 kserver.cc:163] Server-wide thread pool size limit: 3276
15:56:48.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.890805 30569 rpc_server.cc:307] RPC server started. Bound to: 127.29.197.189:46717
15:56:48.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.890878 30643 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.197.189:46717 every 8 connection(s)
15:56:48.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.893451 30569 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-1/data/info.pb
15:56:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.897323 30486 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu as pid 30569
15:56:48.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.897811 30486 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-1/wal/instance
15:56:48.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.899149 30644 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.900934 30486 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-2/wal
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-2/data
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-2/logs
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-2/data/info.pb
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.197.188:41343
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.197.188
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.197.148:40779
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.197.190:35785,127.29.197.189:46717,127.29.197.188:41343
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:56:48.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35269/jwks.json with env {}
15:56:48.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.917088 30644 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.197.190" port: 35785 } has no permanent_uuid. Determining permanent_uuid...
15:56:48.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.941871 30644 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } has no permanent_uuid. Determining permanent_uuid...
15:56:48.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:48.955264 30644 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } has no permanent_uuid. Determining permanent_uuid...
15:56:48.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:48.957756 30591 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.197.188:41343: connect: Connection refused (error 111)
15:56:48.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:48.961995 30644 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.197.188:41343: Network error: Client connection negotiation failed: client connection to 127.29.197.188:41343: connect: Connection refused (error 111)
15:56:49.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:49.011417 30644 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } attempt: 1
15:56:49.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:49.015352 30644 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.197.188:41343: Network error: Client connection negotiation failed: client connection to 127.29.197.188:41343: connect: Connection refused (error 111)
15:56:49.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:49.083701 30644 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } attempt: 2
15:56:49.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:49.087550 30644 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.197.188:41343: Network error: Client connection negotiation failed: client connection to 127.29.197.188:41343: connect: Connection refused (error 111)
15:56:49.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:49.178932 30644 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } attempt: 3
15:56:49.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:49.183584 30644 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.197.188:41343: Network error: Client connection negotiation failed: client connection to 127.29.197.188:41343: connect: Connection refused (error 111)
15:56:49.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:49.215014 30646 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:56:49.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:49.215632 30646 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:56:49.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:49.216509 30646 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:56:49.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:49.246670 30646 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:56:49.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:49.246933 30646 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:56:49.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:49.247135 30646 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:56:49.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:49.247318 30646 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:56:49.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:49.247510 30646 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:56:49.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:49.247654 30646 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35269/jwks.json
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:49.281685 30646 master_runner.cc:386] Master server non-default flags:
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.197.148:40779
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-2/data
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-2/wal
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.197.190:35785,127.29.197.189:46717,127.29.197.188:41343
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.197.188:41343
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35269/jwks.json
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-2/data/info.pb
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.197.188
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-2/logs
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 14:43:21 UTC on a633016ff5c9
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4176
15:56:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:56:49.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:49.282935 30646 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:56:49.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:49.284384 30646 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:56:49.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:49.299399 30660 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:56:49.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:49.326923 30644 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } attempt: 4
15:56:49.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:49.330940 30644 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.197.188:41343: Network error: Client connection negotiation failed: client connection to 127.29.197.188:41343: connect: Connection refused (error 111)
15:56:49.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:49.486757 30567 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } attempt: 7
15:56:49.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:49.501683 30567 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } has no permanent_uuid. Determining permanent_uuid...
15:56:49.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:49.507529 30567 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.197.188:41343: Network error: Client connection negotiation failed: client connection to 127.29.197.188:41343: connect: Connection refused (error 111)
15:56:49.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:49.566105 30567 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } attempt: 1
15:56:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:49.570055 30567 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.197.188:41343: Network error: Client connection negotiation failed: client connection to 127.29.197.188:41343: connect: Connection refused (error 111)
15:56:49.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:49.630510 30644 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } attempt: 5
15:56:49.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:49.634549 30644 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.197.188:41343: Network error: Client connection negotiation failed: client connection to 127.29.197.188:41343: connect: Connection refused (error 111)
15:56:49.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:49.651625 30567 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } attempt: 2
15:56:49.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:49.655565 30567 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.197.188:41343: Network error: Client connection negotiation failed: client connection to 127.29.197.188:41343: connect: Connection refused (error 111)
15:56:49.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:49.741148 30567 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } attempt: 3
15:56:49.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:49.745314 30567 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.197.188:41343: Network error: Client connection negotiation failed: client connection to 127.29.197.188:41343: connect: Connection refused (error 111)
15:56:49.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:49.885735 30567 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } attempt: 4
15:56:49.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:49.889564 30567 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.197.188:41343: Network error: Client connection negotiation failed: client connection to 127.29.197.188:41343: connect: Connection refused (error 111)
15:56:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.172947 30567 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } attempt: 5
15:56:50.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:50.176582 30567 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.197.188:41343: Network error: Client connection negotiation failed: client connection to 127.29.197.188:41343: connect: Connection refused (error 111)
15:56:50.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.182118 30644 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } attempt: 6
15:56:50.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:50.188872 30644 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.29.197.188:41343: Network error: Client connection negotiation failed: client connection to 127.29.197.188:41343: connect: Connection refused (error 111)
15:56:50.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:49.299829 30657 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:56:50.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:49.300737 30658 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:56:50.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:49.301121 30646 server_base.cc:1034] running on GCE node
15:56:50.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.494709 30646 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:56:50.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.497673 30646 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:56:50.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.499120 30646 hybrid_clock.cc:648] HybridClock initialized: now 1736870210499075 us; error 54 us; skew 500 ppm
15:56:50.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.500003 30646 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:56:50.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.509579 30646 webserver.cc:458] Webserver started at http://127.29.197.188:32801/ using document root <none> and password file <none>
15:56:50.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.510468 30646 fs_manager.cc:362] Metadata directory not provided
15:56:50.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.510649 30646 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:56:50.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.511021 30646 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:56:50.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.515964 30646 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-2/data/instance:
15:56:50.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3bf2142e52444155b83a51405b1a48e4"
15:56:50.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 15:56:50 on dist-test-slave-pjn3"
15:56:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.517050 30646 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-2/wal/instance:
15:56:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3bf2142e52444155b83a51405b1a48e4"
15:56:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 15:56:50 on dist-test-slave-pjn3"
15:56:50.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.524003 30646 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
15:56:50.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.529280 30672 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:56:50.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.530279 30646 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
15:56:50.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.530629 30646 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-2/data,/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-2/wal
15:56:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3bf2142e52444155b83a51405b1a48e4"
15:56:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 15:56:50 on dist-test-slave-pjn3"
15:56:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.530949 30646 fs_report.cc:389] FS layout report
15:56:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:56:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-2/wal
15:56:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-2/wal
15:56:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-2/data/data
15:56:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:56:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:56:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:56:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:56:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:56:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:56:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:56:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:56:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:56:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:56:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:56:50.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.582548 30646 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:56:50.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.583921 30646 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:56:50.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.584378 30646 kserver.cc:163] Server-wide thread pool size limit: 3276
15:56:50.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.661028 30646 rpc_server.cc:307] RPC server started. Bound to: 127.29.197.188:41343
15:56:50.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.661126 30725 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.197.188:41343 every 8 connection(s)
15:56:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.663684 30646 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-2/data/info.pb
15:56:50.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.668711 30726 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:56:50.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.672005 30486 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu as pid 30646
15:56:50.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.672463 30486 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/master-2/wal/instance
15:56:50.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.684587 30726 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.197.190" port: 35785 } has no permanent_uuid. Determining permanent_uuid...
15:56:50.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.707719 30726 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } has no permanent_uuid. Determining permanent_uuid...
15:56:50.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.718109 30726 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } has no permanent_uuid. Determining permanent_uuid...
15:56:50.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.729172 30567 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } attempt: 6
15:56:50.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.733448 30726 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4: Bootstrap starting.
15:56:50.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.739266 30726 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4: Neither blocks nor log segments found. Creating new log.
15:56:50.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.741179 30726 log.cc:826] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4: Log is configured to *not* fsync() on all Append() calls
15:56:50.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.746439 30726 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4: No bootstrap required, opened a new log
15:56:50.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.751588 30567 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587: Bootstrap starting.
15:56:50.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.758949 30567 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587: Neither blocks nor log segments found. Creating new log.
15:56:50.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.761255 30567 log.cc:826] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587: Log is configured to *not* fsync() on all Append() calls
15:56:50.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.765959 30567 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587: No bootstrap required, opened a new log
15:56:50.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.776650 30726 raft_consensus.cc:357] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26206da836ae45d898f608eb3d6e5587" member_type: VOTER last_known_addr { host: "127.29.197.190" port: 35785 } } peers { permanent_uuid: "498e4c9ae2b54c1b917287f14f0e1b47" member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } } peers { permanent_uuid: "3bf2142e52444155b83a51405b1a48e4" member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } }
15:56:50.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.777508 30726 raft_consensus.cc:383] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:56:50.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.777812 30726 raft_consensus.cc:738] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3bf2142e52444155b83a51405b1a48e4, State: Initialized, Role: FOLLOWER
15:56:50.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.778678 30726 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26206da836ae45d898f608eb3d6e5587" member_type: VOTER last_known_addr { host: "127.29.197.190" port: 35785 } } peers { permanent_uuid: "498e4c9ae2b54c1b917287f14f0e1b47" member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } } peers { permanent_uuid: "3bf2142e52444155b83a51405b1a48e4" member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } }
15:56:50.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.783370 30737 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26206da836ae45d898f608eb3d6e5587" member_type: VOTER last_known_addr { host: "127.29.197.190" port: 35785 } } peers { permanent_uuid: "498e4c9ae2b54c1b917287f14f0e1b47" member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } } peers { permanent_uuid: "3bf2142e52444155b83a51405b1a48e4" member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } } }
15:56:50.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.784005 30737 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4 [sys.catalog]: This master's current role is: FOLLOWER
15:56:50.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.785359 30726 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4 [sys.catalog]: configured and running, proceeding with master startup.
15:56:50.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.798243 30567 raft_consensus.cc:357] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26206da836ae45d898f608eb3d6e5587" member_type: VOTER last_known_addr { host: "127.29.197.190" port: 35785 } } peers { permanent_uuid: "498e4c9ae2b54c1b917287f14f0e1b47" member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } } peers { permanent_uuid: "3bf2142e52444155b83a51405b1a48e4" member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } }
15:56:50.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.799355 30567 raft_consensus.cc:383] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:56:50.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.799705 30567 raft_consensus.cc:738] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 26206da836ae45d898f608eb3d6e5587, State: Initialized, Role: FOLLOWER
15:56:50.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.800812 30567 consensus_queue.cc:260] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26206da836ae45d898f608eb3d6e5587" member_type: VOTER last_known_addr { host: "127.29.197.190" port: 35785 } } peers { permanent_uuid: "498e4c9ae2b54c1b917287f14f0e1b47" member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } } peers { permanent_uuid: "3bf2142e52444155b83a51405b1a48e4" member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } }
15:56:50.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.804189 30746 sys_catalog.cc:455] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26206da836ae45d898f608eb3d6e5587" member_type: VOTER last_known_addr { host: "127.29.197.190" port: 35785 } } peers { permanent_uuid: "498e4c9ae2b54c1b917287f14f0e1b47" member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } } peers { permanent_uuid: "3bf2142e52444155b83a51405b1a48e4" member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } } }
15:56:50.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.805193 30746 sys_catalog.cc:458] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [sys.catalog]: This master's current role is: FOLLOWER
15:56:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.806524 30567 sys_catalog.cc:564] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [sys.catalog]: configured and running, proceeding with master startup.
15:56:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:50.822803 30749 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:56:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:50.823130 30749 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:56:50.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:50.834853 30521 tablet.cc:2367] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:56:50.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:50.839180 30678 tablet.cc:2367] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:56:50.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.810729 30646 master_runner.cc:186] Error getting master registration for 127.29.197.190:35785: OK, instance_id { permanent_uuid: "26206da836ae45d898f608eb3d6e5587" instance_seqno: 1736870207126756 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:56:50.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.851528 30646 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:56:50.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:50.861368 30761 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:56:50.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:50.861866 30761 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:56:50.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.859344 30498 master_runner.cc:186] Error getting master registration for 127.29.197.189:46717: OK, instance_id { permanent_uuid: "498e4c9ae2b54c1b917287f14f0e1b47" instance_seqno: 1736870208871617 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:56:50.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:50.894418 30498 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:56:51.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.061594 30746 raft_consensus.cc:491] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:56:51.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.062151 30746 raft_consensus.cc:513] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26206da836ae45d898f608eb3d6e5587" member_type: VOTER last_known_addr { host: "127.29.197.190" port: 35785 } } peers { permanent_uuid: "498e4c9ae2b54c1b917287f14f0e1b47" member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } } peers { permanent_uuid: "3bf2142e52444155b83a51405b1a48e4" member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } }
15:56:51.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.066305 30700 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "26206da836ae45d898f608eb3d6e5587" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3bf2142e52444155b83a51405b1a48e4" is_pre_election: true
15:56:51.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.066778 30746 leader_election.cc:290] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 498e4c9ae2b54c1b917287f14f0e1b47 (127.29.197.189:46717), 3bf2142e52444155b83a51405b1a48e4 (127.29.197.188:41343)
15:56:51.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.067202 30700 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 26206da836ae45d898f608eb3d6e5587 in term 0.
15:56:51.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.068616 30516 leader_election.cc:304] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26206da836ae45d898f608eb3d6e5587, 3bf2142e52444155b83a51405b1a48e4; no voters:
15:56:51.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.069520 30746 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:56:51.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.069864 30746 raft_consensus.cc:491] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:56:51.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.070186 30746 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [term 0 FOLLOWER]: Advancing to term 1
15:56:51.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.076552 30746 raft_consensus.cc:513] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26206da836ae45d898f608eb3d6e5587" member_type: VOTER last_known_addr { host: "127.29.197.190" port: 35785 } } peers { permanent_uuid: "498e4c9ae2b54c1b917287f14f0e1b47" member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } } peers { permanent_uuid: "3bf2142e52444155b83a51405b1a48e4" member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } }
15:56:51.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.078346 30746 leader_election.cc:290] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [CANDIDATE]: Term 1 election: Requested vote from peers 498e4c9ae2b54c1b917287f14f0e1b47 (127.29.197.189:46717), 3bf2142e52444155b83a51405b1a48e4 (127.29.197.188:41343)
15:56:51.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.079882 30700 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "26206da836ae45d898f608eb3d6e5587" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3bf2142e52444155b83a51405b1a48e4"
15:56:51.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.080468 30700 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4 [term 0 FOLLOWER]: Advancing to term 1
15:56:51.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.087165 30700 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26206da836ae45d898f608eb3d6e5587 in term 1.
15:56:51.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.088109 30516 leader_election.cc:304] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 26206da836ae45d898f608eb3d6e5587, 3bf2142e52444155b83a51405b1a48e4; no voters:
15:56:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.088960 30746 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [term 1 FOLLOWER]: Leader election won for term 1
15:56:51.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.096333 30746 raft_consensus.cc:695] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [term 1 LEADER]: Becoming Leader. State: Replica: 26206da836ae45d898f608eb3d6e5587, State: Running, Role: LEADER
15:56:51.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.097225 30746 consensus_queue.cc:237] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26206da836ae45d898f608eb3d6e5587" member_type: VOTER last_known_addr { host: "127.29.197.190" port: 35785 } } peers { permanent_uuid: "498e4c9ae2b54c1b917287f14f0e1b47" member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } } peers { permanent_uuid: "3bf2142e52444155b83a51405b1a48e4" member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } }
15:56:51.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.105103 30763 sys_catalog.cc:455] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 26206da836ae45d898f608eb3d6e5587. Latest consensus state: current_term: 1 leader_uuid: "26206da836ae45d898f608eb3d6e5587" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26206da836ae45d898f608eb3d6e5587" member_type: VOTER last_known_addr { host: "127.29.197.190" port: 35785 } } peers { permanent_uuid: "498e4c9ae2b54c1b917287f14f0e1b47" member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } } peers { permanent_uuid: "3bf2142e52444155b83a51405b1a48e4" member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } } }
15:56:51.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.105908 30763 sys_catalog.cc:458] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [sys.catalog]: This master's current role is: LEADER
15:56:51.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.111507 30765 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
15:56:51.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.118564 30765 catalog_manager.cc:1485] Initializing Kudu cluster ID...
15:56:51.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.141495 30700 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4 [term 1 FOLLOWER]: Refusing update from remote peer 26206da836ae45d898f608eb3d6e5587: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:56:51.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.143222 30763 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3bf2142e52444155b83a51405b1a48e4" member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:56:51.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.193560 30737 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 26206da836ae45d898f608eb3d6e5587. Latest consensus state: current_term: 1 leader_uuid: "26206da836ae45d898f608eb3d6e5587" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26206da836ae45d898f608eb3d6e5587" member_type: VOTER last_known_addr { host: "127.29.197.190" port: 35785 } } peers { permanent_uuid: "498e4c9ae2b54c1b917287f14f0e1b47" member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } } peers { permanent_uuid: "3bf2142e52444155b83a51405b1a48e4" member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } } }
15:56:51.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.194274 30737 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4 [sys.catalog]: This master's current role is: FOLLOWER
15:56:51.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.201213 30763 sys_catalog.cc:455] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "26206da836ae45d898f608eb3d6e5587" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26206da836ae45d898f608eb3d6e5587" member_type: VOTER last_known_addr { host: "127.29.197.190" port: 35785 } } peers { permanent_uuid: "498e4c9ae2b54c1b917287f14f0e1b47" member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } } peers { permanent_uuid: "3bf2142e52444155b83a51405b1a48e4" member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } } }
15:56:51.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.202420 30763 sys_catalog.cc:458] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [sys.catalog]: This master's current role is: LEADER
15:56:51.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.215453 30737 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "26206da836ae45d898f608eb3d6e5587" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26206da836ae45d898f608eb3d6e5587" member_type: VOTER last_known_addr { host: "127.29.197.190" port: 35785 } } peers { permanent_uuid: "498e4c9ae2b54c1b917287f14f0e1b47" member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } } peers { permanent_uuid: "3bf2142e52444155b83a51405b1a48e4" member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } } }
15:56:51.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.216176 30737 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4 [sys.catalog]: This master's current role is: FOLLOWER
15:56:51.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.229837 30765 catalog_manager.cc:1348] Generated new cluster ID: d4c95c0f179c4918a691d22cb4a5f19e
15:56:51.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.230146 30765 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
15:56:51.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.249385 30644 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } attempt: 7
15:56:51.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.285177 30765 catalog_manager.cc:1371] Generated new certificate authority record
15:56:51.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.287163 30765 catalog_manager.cc:1505] Loading token signing keys...
15:56:51.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.318079 30644 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 498e4c9ae2b54c1b917287f14f0e1b47: Bootstrap starting.
15:56:51.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.324594 30765 catalog_manager.cc:5899] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587: Generated new TSK 0
15:56:51.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.325577 30644 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 498e4c9ae2b54c1b917287f14f0e1b47: Neither blocks nor log segments found. Creating new log.
15:56:51.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.325654 30765 catalog_manager.cc:1515] Initializing in-progress tserver states...
15:56:51.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.327986 30644 log.cc:826] T 00000000000000000000000000000000 P 498e4c9ae2b54c1b917287f14f0e1b47: Log is configured to *not* fsync() on all Append() calls
15:56:51.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.343973 30644 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 498e4c9ae2b54c1b917287f14f0e1b47: No bootstrap required, opened a new log
15:56:51.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.349331 30486 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu
15:56:51.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu
15:56:51.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-0/wal
15:56:51.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-0/data
15:56:51.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:56:51.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:56:51.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:56:51.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:56:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:56:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:56:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-0/logs
15:56:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-0/data/info.pb
15:56:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:56:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:56:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:56:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:56:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:56:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:56:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:56:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:56:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:56:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:56:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.197.129:0
15:56:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.197.129
15:56:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.197.129
15:56:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:56:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.197.190:35785,127.29.197.189:46717,127.29.197.188:41343
15:56:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.197.148:40779
15:56:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:56:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:56:51.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:56:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.397600 30644 raft_consensus.cc:357] T 00000000000000000000000000000000 P 498e4c9ae2b54c1b917287f14f0e1b47 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26206da836ae45d898f608eb3d6e5587" member_type: VOTER last_known_addr { host: "127.29.197.190" port: 35785 } } peers { permanent_uuid: "498e4c9ae2b54c1b917287f14f0e1b47" member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } } peers { permanent_uuid: "3bf2142e52444155b83a51405b1a48e4" member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } }
15:56:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.398483 30644 raft_consensus.cc:383] T 00000000000000000000000000000000 P 498e4c9ae2b54c1b917287f14f0e1b47 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:56:51.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.398806 30644 raft_consensus.cc:738] T 00000000000000000000000000000000 P 498e4c9ae2b54c1b917287f14f0e1b47 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 498e4c9ae2b54c1b917287f14f0e1b47, State: Initialized, Role: FOLLOWER
15:56:51.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.399717 30644 consensus_queue.cc:260] T 00000000000000000000000000000000 P 498e4c9ae2b54c1b917287f14f0e1b47 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26206da836ae45d898f608eb3d6e5587" member_type: VOTER last_known_addr { host: "127.29.197.190" port: 35785 } } peers { permanent_uuid: "498e4c9ae2b54c1b917287f14f0e1b47" member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } } peers { permanent_uuid: "3bf2142e52444155b83a51405b1a48e4" member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } }
15:56:51.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.411732 30776 sys_catalog.cc:455] T 00000000000000000000000000000000 P 498e4c9ae2b54c1b917287f14f0e1b47 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26206da836ae45d898f608eb3d6e5587" member_type: VOTER last_known_addr { host: "127.29.197.190" port: 35785 } } peers { permanent_uuid: "498e4c9ae2b54c1b917287f14f0e1b47" member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } } peers { permanent_uuid: "3bf2142e52444155b83a51405b1a48e4" member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } } }
15:56:51.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.413628 30776 sys_catalog.cc:458] T 00000000000000000000000000000000 P 498e4c9ae2b54c1b917287f14f0e1b47 [sys.catalog]: This master's current role is: FOLLOWER
15:56:51.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.416049 30644 sys_catalog.cc:564] T 00000000000000000000000000000000 P 498e4c9ae2b54c1b917287f14f0e1b47 [sys.catalog]: configured and running, proceeding with master startup.
15:56:51.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.416246 30616 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "26206da836ae45d898f608eb3d6e5587" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "498e4c9ae2b54c1b917287f14f0e1b47"
15:56:51.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.416999 30616 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 498e4c9ae2b54c1b917287f14f0e1b47 [term 0 FOLLOWER]: Advancing to term 1
15:56:51.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.421790 30617 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "26206da836ae45d898f608eb3d6e5587" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "498e4c9ae2b54c1b917287f14f0e1b47" is_pre_election: true
15:56:51.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.428189 30616 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 498e4c9ae2b54c1b917287f14f0e1b47 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 26206da836ae45d898f608eb3d6e5587 in term 1.
15:56:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.429646 30615 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 498e4c9ae2b54c1b917287f14f0e1b47 [term 1 FOLLOWER]: Refusing update from remote peer 26206da836ae45d898f608eb3d6e5587: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:56:51.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.436595 30763 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [LEADER]: Connected to new peer: Peer: permanent_uuid: "498e4c9ae2b54c1b917287f14f0e1b47" member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:56:51.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.490003 30776 sys_catalog.cc:455] T 00000000000000000000000000000000 P 498e4c9ae2b54c1b917287f14f0e1b47 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 26206da836ae45d898f608eb3d6e5587. Latest consensus state: current_term: 1 leader_uuid: "26206da836ae45d898f608eb3d6e5587" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26206da836ae45d898f608eb3d6e5587" member_type: VOTER last_known_addr { host: "127.29.197.190" port: 35785 } } peers { permanent_uuid: "498e4c9ae2b54c1b917287f14f0e1b47" member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } } peers { permanent_uuid: "3bf2142e52444155b83a51405b1a48e4" member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } } }
15:56:51.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.490738 30776 sys_catalog.cc:458] T 00000000000000000000000000000000 P 498e4c9ae2b54c1b917287f14f0e1b47 [sys.catalog]: This master's current role is: FOLLOWER
15:56:51.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.492167 30785 sys_catalog.cc:455] T 00000000000000000000000000000000 P 498e4c9ae2b54c1b917287f14f0e1b47 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "26206da836ae45d898f608eb3d6e5587" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26206da836ae45d898f608eb3d6e5587" member_type: VOTER last_known_addr { host: "127.29.197.190" port: 35785 } } peers { permanent_uuid: "498e4c9ae2b54c1b917287f14f0e1b47" member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } } peers { permanent_uuid: "3bf2142e52444155b83a51405b1a48e4" member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } } }
15:56:51.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.493211 30785 sys_catalog.cc:458] T 00000000000000000000000000000000 P 498e4c9ae2b54c1b917287f14f0e1b47 [sys.catalog]: This master's current role is: FOLLOWER
15:56:51.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.506348 30792 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:56:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.827857 30749 catalog_manager.cc:1260] Loaded cluster ID: d4c95c0f179c4918a691d22cb4a5f19e
15:56:51.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.828337 30749 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4: loading cluster ID for follower catalog manager: success
15:56:51.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.835346 30749 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4: acquiring CA information for follower catalog manager: success
15:56:51.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.840608 30749 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 3bf2142e52444155b83a51405b1a48e4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:56:51.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:51.862130 30775 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:56:51.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:51.862913 30775 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:56:51.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:51.863699 30775 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:56:51.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:51.920558 30775 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:56:51.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:51.922219 30775 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.197.129
15:56:51.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.946090 30763 sys_catalog.cc:455] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "26206da836ae45d898f608eb3d6e5587" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "26206da836ae45d898f608eb3d6e5587" member_type: VOTER last_known_addr { host: "127.29.197.190" port: 35785 } } peers { permanent_uuid: "498e4c9ae2b54c1b917287f14f0e1b47" member_type: VOTER last_known_addr { host: "127.29.197.189" port: 46717 } } peers { permanent_uuid: "3bf2142e52444155b83a51405b1a48e4" member_type: VOTER last_known_addr { host: "127.29.197.188" port: 41343 } } }
15:56:51.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.948575 30763 sys_catalog.cc:458] T 00000000000000000000000000000000 P 26206da836ae45d898f608eb3d6e5587 [sys.catalog]: This master's current role is: LEADER
15:56:51.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.987123 30775 tablet_server_runner.cc:78] Tablet server non-default flags:
15:56:51.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.197.148:40779
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-0/data
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-0/wal
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.197.129:0
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-0/data/info.pb
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.197.129
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.197.190:35785,127.29.197.189:46717,127.29.197.188:41343
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.197.129
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-0/logs
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 14:43:21 UTC on a633016ff5c9
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4176
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:56:51.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.988790 30775 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:56:51.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:51.990978 30775 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:56:52.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:52.010708 30803 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:56:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:52.522238 30792 catalog_manager.cc:1260] Loaded cluster ID: d4c95c0f179c4918a691d22cb4a5f19e
15:56:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:52.522517 30792 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 498e4c9ae2b54c1b917287f14f0e1b47: loading cluster ID for follower catalog manager: success
15:56:52.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:52.531345 30792 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 498e4c9ae2b54c1b917287f14f0e1b47: acquiring CA information for follower catalog manager: success
15:56:52.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:52.544723 30792 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 498e4c9ae2b54c1b917287f14f0e1b47: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:56:53.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:53.657196 30563 debug-util.cc:398] Leaking SignalData structure 0x7b08000c5340 after lost signal to thread 30499
15:56:53.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:53.658183 30563 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8d60 after lost signal to thread 30566
15:56:54.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:53.409027 30802 debug-util.cc:398] Leaking SignalData structure 0x7b0800006bc0 after lost signal to thread 30775
15:56:54.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:52.013581 30804 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:56:54.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:54.972787 30775 thread.cc:635] OpenStack (cloud detector) Time spent creating pthread: real 2.960s user 0.793s sys 1.436s
15:56:54.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:54.974196 30805 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2960 milliseconds
15:56:54.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:54.974527 30775 thread.cc:601] OpenStack (cloud detector) Time spent starting thread: real 2.962s user 0.794s sys 1.437s
15:56:54.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:54.976480 30806 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:56:54.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:54.976431 30775 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:56:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:54.979774 30775 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:56:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:54.984597 30775 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:56:54.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:54.986050 30775 hybrid_clock.cc:648] HybridClock initialized: now 1736870214985944 us; error 116 us; skew 500 ppm
15:56:54.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:54.986873 30775 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:56:54.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:54.994167 30775 webserver.cc:458] Webserver started at http://127.29.197.129:42497/ using document root <none> and password file <none>
15:56:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:54.995074 30775 fs_manager.cc:362] Metadata directory not provided
15:56:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:54.995291 30775 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:56:54.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:54.995738 30775 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:56:55.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.001087 30775 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-0/data/instance:
15:56:55.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "422cd83385e14a0ab15a3ce97d7c0b80"
15:56:55.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 15:56:54 on dist-test-slave-pjn3"
15:56:55.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.002206 30775 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-0/wal/instance:
15:56:55.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "422cd83385e14a0ab15a3ce97d7c0b80"
15:56:55.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 15:56:54 on dist-test-slave-pjn3"
15:56:55.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.009722 30775 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.002s
15:56:55.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.015534 30814 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:56:55.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.016597 30775 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
15:56:55.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.016901 30775 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-0/data,/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-0/wal
15:56:55.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "422cd83385e14a0ab15a3ce97d7c0b80"
15:56:55.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 15:56:54 on dist-test-slave-pjn3"
15:56:55.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.017213 30775 fs_report.cc:389] FS layout report
15:56:55.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:56:55.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-0/wal
15:56:55.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-0/wal
15:56:55.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-0/data/data
15:56:55.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:56:55.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:56:55.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:56:55.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:56:55.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:56:55.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:56:55.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:56:55.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:56:55.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:56:55.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:56:55.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:56:55.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.071389 30775 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:56:55.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.072858 30775 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:56:55.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.073323 30775 kserver.cc:163] Server-wide thread pool size limit: 3276
15:56:55.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.076333 30775 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:56:55.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.081152 30775 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:56:55.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.081358 30775 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:56:55.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.081678 30775 ts_tablet_manager.cc:610] Registered 0 tablets
15:56:55.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.081844 30775 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:56:55.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.306180 30775 rpc_server.cc:307] RPC server started. Bound to: 127.29.197.129:43611
15:56:55.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.306407 30928 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.197.129:43611 every 8 connection(s)
15:56:55.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.309572 30775 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-0/data/info.pb
15:56:55.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.314419 30486 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu as pid 30775
15:56:55.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.315096 30486 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-0/wal/instance
15:56:55.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.334352 30486 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu
15:56:55.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu
15:56:55.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-1/wal
15:56:55.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-1/data
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-1/logs
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-1/data/info.pb
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.197.130:0
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.197.130
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.197.130
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.197.190:35785,127.29.197.189:46717,127.29.197.188:41343
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.197.148:40779
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:56:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:56:55.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.379055 30930 heartbeater.cc:346] Connected to a master server at 127.29.197.188:41343
15:56:55.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.379565 30930 heartbeater.cc:463] Registering TS with master...
15:56:55.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.388106 30930 heartbeater.cc:510] Master 127.29.197.188:41343 requested a full tablet report, sending...
15:56:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.391856 30690 ts_manager.cc:194] Registered new tserver with Master: 422cd83385e14a0ab15a3ce97d7c0b80 (127.29.197.129:43611)
15:56:55.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.404464 30931 heartbeater.cc:346] Connected to a master server at 127.29.197.189:46717
15:56:55.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.404839 30931 heartbeater.cc:463] Registering TS with master...
15:56:55.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.405890 30931 heartbeater.cc:510] Master 127.29.197.189:46717 requested a full tablet report, sending...
15:56:55.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.408953 30607 ts_manager.cc:194] Registered new tserver with Master: 422cd83385e14a0ab15a3ce97d7c0b80 (127.29.197.129:43611)
15:56:55.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.421334 30936 heartbeater.cc:346] Connected to a master server at 127.29.197.190:35785
15:56:55.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.421722 30936 heartbeater.cc:463] Registering TS with master...
15:56:55.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.422569 30936 heartbeater.cc:510] Master 127.29.197.190:35785 requested a full tablet report, sending...
15:56:55.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.425097 30531 ts_manager.cc:194] Registered new tserver with Master: 422cd83385e14a0ab15a3ce97d7c0b80 (127.29.197.129:43611)
15:56:55.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.428007 30531 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.29.197.129:41209
15:56:55.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:55.805897 30935 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:56:55.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:55.806416 30935 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:56:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:55.806950 30935 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:56:55.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:55.839030 30935 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:56:55.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:55.839967 30935 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.197.130
15:56:55.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.906183 30935 tablet_server_runner.cc:78] Tablet server non-default flags:
15:56:55.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:56:55.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.197.148:40779
15:56:55.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:56:55.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:56:55.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-1/data
15:56:55.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-1/wal
15:56:55.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:56:55.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:56:55.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.197.130:0
15:56:55.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:56:55.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:56:55.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:56:55.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-1/data/info.pb
15:56:55.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.197.130
15:56:55.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:56:55.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.197.190:35785,127.29.197.189:46717,127.29.197.188:41343
15:56:55.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:56:55.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:56:55.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:56:55.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:56:55.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:56:55.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.197.130
15:56:55.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-1/logs
15:56:55.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:56:55.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:56:55.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:56:55.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:56:55.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:56:55.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
15:56:55.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:56:55.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 14:43:21 UTC on a633016ff5c9
15:56:55.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4176
15:56:55.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:56:55.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.909796 30935 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:56:55.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:55.912225 30935 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:56:55.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:55.964943 30949 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:56:56.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:56.432000 30936 heartbeater.cc:502] Master 127.29.197.190:35785 was elected leader, sending a full tablet report...
15:56:58.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:57.360778 30948 debug-util.cc:398] Leaking SignalData structure 0x7b0800006bc0 after lost signal to thread 30935
15:56:58.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:58.487995 30948 kernel_stack_watchdog.cc:198] Thread 30935 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:636 for 391ms:
15:56:58.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:56:58.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:56:58.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:56:58.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:56:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:55.967653 30950 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:56:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:58.489373 30935 thread.cc:635] OpenStack (cloud detector) Time spent creating pthread: real 2.522s user 0.863s sys 1.530s
15:56:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:58.489799 30935 thread.cc:601] OpenStack (cloud detector) Time spent starting thread: real 2.523s user 0.864s sys 1.530s
15:56:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:58.491931 30952 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:56:58.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:58.493713 30951 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2522 milliseconds
15:56:58.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.493703 30935 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:56:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.495028 30935 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:56:58.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.497102 30935 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:56:58.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.498517 30935 hybrid_clock.cc:648] HybridClock initialized: now 1736870218498470 us; error 45 us; skew 500 ppm
15:56:58.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.499352 30935 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:56:58.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.510609 30935 webserver.cc:458] Webserver started at http://127.29.197.130:35079/ using document root <none> and password file <none>
15:56:58.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.511483 30935 fs_manager.cc:362] Metadata directory not provided
15:56:58.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.511668 30935 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:56:58.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.512034 30935 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:56:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.516696 30935 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-1/data/instance:
15:56:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "163b3417460e4e87b7b7ce068e47eb4a"
15:56:58.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 15:56:58 on dist-test-slave-pjn3"
15:56:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.517803 30935 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-1/wal/instance:
15:56:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "163b3417460e4e87b7b7ce068e47eb4a"
15:56:58.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 15:56:58 on dist-test-slave-pjn3"
15:56:58.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.524670 30935 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
15:56:58.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.530172 30962 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:56:58.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.531133 30935 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
15:56:58.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.531421 30935 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-1/data,/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-1/wal
15:56:58.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "163b3417460e4e87b7b7ce068e47eb4a"
15:56:58.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 15:56:58 on dist-test-slave-pjn3"
15:56:58.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.531754 30935 fs_report.cc:389] FS layout report
15:56:58.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:56:58.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-1/wal
15:56:58.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-1/wal
15:56:58.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-1/data/data
15:56:58.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:56:58.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:56:58.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:56:58.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:56:58.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:56:58.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:56:58.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:56:58.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:56:58.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:56:58.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:56:58.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:56:58.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.585700 30935 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:56:58.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.587141 30935 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:56:58.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.587550 30935 kserver.cc:163] Server-wide thread pool size limit: 3276
15:56:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.590178 30935 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:56:58.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.594262 30935 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:56:58.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.594465 30935 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:56:58.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.594658 30935 ts_tablet_manager.cc:610] Registered 0 tablets
15:56:58.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.594790 30935 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:56:58.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.734380 30935 rpc_server.cc:307] RPC server started. Bound to: 127.29.197.130:41327
15:56:58.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.734499 31076 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.197.130:41327 every 8 connection(s)
15:56:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.736783 30935 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-1/data/info.pb
15:56:58.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.741415 30486 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu as pid 30935
15:56:58.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.741886 30486 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-1/wal/instance
15:56:58.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.757298 30486 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu
15:56:58.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu
15:56:58.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-2/wal
15:56:58.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-2/data
15:56:58.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:56:58.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:56:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:56:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:56:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:56:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:56:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-2/logs
15:56:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-2/data/info.pb
15:56:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:56:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:56:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:56:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:56:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:56:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:56:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:56:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:56:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:56:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:56:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.197.131:0
15:56:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.197.131
15:56:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.197.131
15:56:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:56:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.197.190:35785,127.29.197.189:46717,127.29.197.188:41343
15:56:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.197.148:40779
15:56:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:56:58.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:56:58.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:56:58.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.787927 31078 heartbeater.cc:346] Connected to a master server at 127.29.197.189:46717
15:56:58.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.788439 31078 heartbeater.cc:463] Registering TS with master...
15:56:58.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.789791 31078 heartbeater.cc:510] Master 127.29.197.189:46717 requested a full tablet report, sending...
15:56:58.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.789811 31077 heartbeater.cc:346] Connected to a master server at 127.29.197.188:41343
15:56:58.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.790786 31077 heartbeater.cc:463] Registering TS with master...
15:56:58.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.791656 31077 heartbeater.cc:510] Master 127.29.197.188:41343 requested a full tablet report, sending...
15:56:58.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.792449 31079 heartbeater.cc:346] Connected to a master server at 127.29.197.190:35785
15:56:58.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.792804 31079 heartbeater.cc:463] Registering TS with master...
15:56:58.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.793354 30607 ts_manager.cc:194] Registered new tserver with Master: 163b3417460e4e87b7b7ce068e47eb4a (127.29.197.130:41327)
15:56:58.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.793695 31079 heartbeater.cc:510] Master 127.29.197.190:35785 requested a full tablet report, sending...
15:56:58.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.794389 30690 ts_manager.cc:194] Registered new tserver with Master: 163b3417460e4e87b7b7ce068e47eb4a (127.29.197.130:41327)
15:56:58.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.796609 30531 ts_manager.cc:194] Registered new tserver with Master: 163b3417460e4e87b7b7ce068e47eb4a (127.29.197.130:41327)
15:56:58.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:58.798434 30531 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.29.197.130:37869
15:56:59.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:59.096433 31087 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:56:59.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:59.096953 31087 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:56:59.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:59.097554 31087 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:56:59.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:59.130208 31087 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:56:59.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:59.131439 31087 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.197.131
15:56:59.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:59.166297 31087 tablet_server_runner.cc:78] Tablet server non-default flags:
15:56:59.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:56:59.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.197.148:40779
15:56:59.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:56:59.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:56:59.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-2/data
15:56:59.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-2/wal
15:56:59.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:56:59.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.197.131:0
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-2/data/info.pb
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.197.131
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.197.190:35785,127.29.197.189:46717,127.29.197.188:41343
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.197.131
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-2/logs
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 14:43:21 UTC on a633016ff5c9
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4176
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:56:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:59.167662 31087 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:56:59.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:59.169183 31087 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:56:59.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:59.193346 31093 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:56:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:56:59.801206 31079 heartbeater.cc:502] Master 127.29.197.190:35785 was elected leader, sending a full tablet report...
15:57:00.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==31087==WARNING: Can't read from symbolizer at fd 5
15:57:00.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==31087==WARNING: invalid path to external symbolizer!
15:57:00.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==31087==WARNING: Can't write to symbolizer at fd 10
15:57:00.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==31087==WARNING: Can't write to symbolizer at fd 10
15:57:00.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==31087==WARNING: Can't write to symbolizer at fd 10
15:57:00.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==31087==WARNING: Failed to use and restart external symbolizer!
15:57:00.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=31087)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 8 at 0x7b0400006d90 by thread T6:
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x4964bf)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libc.so.6+0xd3fe4)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libglog.so.1+0x1f60b)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libglog.so.1+0x17b16)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libglog.so.1+0x18f10)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libkudu_cloud_util.so+0x11971)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libkudu_cloud_util.so+0x12c8d)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libkudu_cloud_util.so+0x12c19)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libkudu_cloud_util.so+0x12ba9)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (libkudu_cloud_util.so+0x12b71)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (libkudu_cloud_util.so+0x11e6d)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x59ea04)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x59e839)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_util.so+0x448fe6)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x4689ff)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous write of size 8 at 0x7b0400006d90 by thread T5:
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x4933dc)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libc.so.6+0x9daf9)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libglog.so.1+0x1f60b)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libglog.so.1+0x17b16)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libglog.so.1+0x18f10)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libkudu_cloud_util.so+0x11971)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libkudu_cloud_util.so+0x12c8d)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libkudu_cloud_util.so+0x12c19)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libkudu_cloud_util.so+0x12ba9)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (libkudu_cloud_util.so+0x12b71)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (libkudu_cloud_util.so+0x11e6d)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x59ea04)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x59e839)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_util.so+0x448fe6)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x4689ff)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T6 'Azure-31094' (tid=31094, running) created by main thread at:
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x46c4b5)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libkudu_util.so+0x44884a)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (kudu+0x58ec29)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libkudu_cloud_util.so+0x113f5)
15:57:00.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libserver_process.so+0x145116)
15:57:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libserver_process.so+0x143576)
15:57:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libkserver.so+0x12a01)
15:57:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libtserver.so+0x255b52)
15:57:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libtserver.so+0x263451)
15:57:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x629859)
15:57:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x529a01)
15:57:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x52996c)
15:57:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x5298dc)
15:57:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x528545)
15:57:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x13df8f)
15:57:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (libkudu_tools_util.so+0x1307f1)
15:57:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (libkudu_tools_util.so+0x12e582)
15:57:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x6455a0)
15:57:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (kudu+0x645ed7)
15:57:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (kudu+0x64641e)
15:57:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (libc.so.6+0x21bf6)
15:57:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T5 'AWS-31093' (tid=31093, running) created by main thread at:
15:57:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x46c4b5)
15:57:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libkudu_util.so+0x44884a)
15:57:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (kudu+0x58ec29)
15:57:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libkudu_cloud_util.so+0x113f5)
15:57:00.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libserver_process.so+0x145116)
15:57:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libserver_process.so+0x143576)
15:57:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libkserver.so+0x12a01)
15:57:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libtserver.so+0x255b52)
15:57:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libtserver.so+0x263451)
15:57:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x629859)
15:57:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x529a01)
15:57:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x52996c)
15:57:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x5298dc)
15:57:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x528545)
15:57:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x13df8f)
15:57:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (libkudu_tools_util.so+0x1307f1)
15:57:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (libkudu_tools_util.so+0x12e582)
15:57:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x6455a0)
15:57:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (kudu+0x645ed7)
15:57:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (kudu+0x64641e)
15:57:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (libc.so.6+0x21bf6)
15:57:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu+0x4964bf)
15:57:00.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
15:57:00.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:59.194613 31096 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:57:00.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
15:57:00.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:56:59.193995 31094 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:57:00.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=31087)
15:57:00.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Read of size 8 at 0x7fe657ffd5f0 by thread T8:
15:57:00.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libkudu_util.so+0x4491f5)
15:57:00.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libkudu_util.so+0x4534ac)
15:57:00.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libkudu_util.so+0x449002)
15:57:00.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (kudu+0x4689ff)
15:57:00.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:00.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous write of size 8 at 0x7fe657ffd5f0 by thread T5 (mutexes: write M83):
15:57:00.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libglog.so.1+0x2b07e)
15:57:00.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libkudu_util.so+0x449230)
15:57:00.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libkudu_util.so+0x4534ac)
15:57:00.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libkudu_util.so+0x449002)
15:57:00.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (kudu+0x4689ff)
15:57:00.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:00.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Location is global '<null>' at 0x000000000000 (libkudu_util.so+0x0000004f65f0)
15:57:00.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:00.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mutex M83 (0x7fe655f37840) created at:
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x49cc9b)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libglog.so.1+0xe6ec)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (ld-linux-x86-64.so.2+0x108f2)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T8 'OpenStack-31096' (tid=31096, running) created by main thread at:
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x46c4b5)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libkudu_util.so+0x44884a)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (kudu+0x58ec29)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libkudu_cloud_util.so+0x113f5)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libserver_process.so+0x145116)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libserver_process.so+0x143576)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libkserver.so+0x12a01)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libtserver.so+0x255b52)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libtserver.so+0x263451)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x629859)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x529a01)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x52996c)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x5298dc)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x528545)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x13df8f)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (libkudu_tools_util.so+0x1307f1)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (libkudu_tools_util.so+0x12e582)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x6455a0)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (kudu+0x645ed7)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (kudu+0x64641e)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (libc.so.6+0x21bf6)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T5 'AWS-31093' (tid=31093, finished) created by main thread at:
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x46c4b5)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libkudu_util.so+0x44884a)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (kudu+0x58ec29)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libkudu_cloud_util.so+0x113f5)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libserver_process.so+0x145116)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libserver_process.so+0x143576)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libkserver.so+0x12a01)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libtserver.so+0x255b52)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libtserver.so+0x263451)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x629859)
15:57:00.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x529a01)
15:57:00.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x52996c)
15:57:00.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x5298dc)
15:57:00.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x528545)
15:57:00.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x13df8f)
15:57:00.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (libkudu_tools_util.so+0x1307f1)
15:57:00.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (libkudu_tools_util.so+0x12e582)
15:57:00.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x6455a0)
15:57:00.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (kudu+0x645ed7)
15:57:00.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (kudu+0x64641e)
15:57:00.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (libc.so.6+0x21bf6)
15:57:00.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:00.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-taskhTbTJZ/build/tsan/lib/libkudu_util.so+0x4491f5)
15:57:00.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
15:57:00.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:57:00.282250 31095 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1089 milliseconds
15:57:00.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:57:00.282332 31087 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:57:00.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:57:00.283492 31087 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:57:00.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:57:00.286166 31087 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:57:00.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:57:00.287573 31087 hybrid_clock.cc:648] HybridClock initialized: now 1736870220287532 us; error 54 us; skew 500 ppm
15:57:00.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:57:00.288322 31087 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:57:00.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:57:00.295004 31087 webserver.cc:458] Webserver started at http://127.29.197.131:34969/ using document root <none> and password file <none>
15:57:00.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:57:00.295922 31087 fs_manager.cc:362] Metadata directory not provided
15:57:00.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:57:00.296113 31087 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:57:00.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:57:00.296315 31087 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:57:00.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:57:00.296854 31087 fs_manager.cc:362] Metadata directory not provided
15:57:00.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:57:00.297001 31087 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:57:00.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:57:00.297609 31087 ts_tablet_manager.cc:1500] Shutting down tablet manager...
15:57:00.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
15:57:00.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=31087)
15:57:00.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 4 at 0x7b6800003c00 by main thread (mutexes: write M24624009580658040):
15:57:00.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1becb4)
15:57:00.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x18e0c2)
15:57:00.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x142eb2)
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libserver_process.so+0x142c46)
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmaster.so+0x3fcc3f)
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libtserver.so+0x2554e2)
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libtserver.so+0x2634da)
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x629859)
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x529a01)
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x52996c)
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x5298dc)
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x528545)
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x13df8f)
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x1307f1)
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x12e582)
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6455a0)
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x645ed7)
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x64641e)
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libc.so.6+0x21bf6)
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous read of size 4 at 0x7b6800003c00 by thread T12:
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1c069f)
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (kudu+0x4689ff)
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Location is heap block of size 1368 at 0x7b6800003c00 allocated by main thread:
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x498c1c)
15:57:00.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1bf1c4)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18f020)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libserver_process.so+0x143a6e)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libkserver.so+0x12a01)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libtserver.so+0x255b52)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libtserver.so+0x263451)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x629859)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x529a01)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x52996c)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x5298dc)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x528545)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x13df8f)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x1307f1)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x12e582)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6455a0)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x645ed7)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x64641e)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libc.so.6+0x21bf6)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mutex M24624009580658040 is already destroyed.
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T12 'sq_acceptor' (tid=31103, running) created by main thread at:
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x46c4b5)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c0477)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18f020)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libserver_process.so+0x143a6e)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libkserver.so+0x12a01)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libtserver.so+0x255b52)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libtserver.so+0x263451)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x629859)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x529a01)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x52996c)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x5298dc)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x528545)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x13df8f)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x1307f1)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x12e582)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6455a0)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x645ed7)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x64641e)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libc.so.6+0x21bf6)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-taskhTbTJZ/build/tsan/lib/libserver_process.so+0x1becb4)
15:57:00.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
15:57:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
15:57:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=31087)
15:57:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 8 at 0x7b0c00040200 by thread T12:
15:57:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x4964bf)
15:57:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c0730)
15:57:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (kudu+0x4689ff)
15:57:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous read of size 4 at 0x7b0c00040200 by main thread:
15:57:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1b9a98)
15:57:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x18fbdb)
15:57:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18f108)
15:57:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libserver_process.so+0x143a6e)
15:57:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libkserver.so+0x12a01)
15:57:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libtserver.so+0x255b52)
15:57:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libtserver.so+0x263451)
15:57:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x629859)
15:57:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x529a01)
15:57:00.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x52996c)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x5298dc)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x528545)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x13df8f)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x1307f1)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x12e582)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6455a0)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x645ed7)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x64641e)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libc.so.6+0x21bf6)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T12 'sq_acceptor' (tid=31103, running) created by main thread at:
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x46c4b5)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c0477)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18f020)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libserver_process.so+0x143a6e)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libkserver.so+0x12a01)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libtserver.so+0x255b52)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libtserver.so+0x263451)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x629859)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x529a01)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x52996c)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x5298dc)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x528545)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x13df8f)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x1307f1)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x12e582)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6455a0)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x645ed7)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x64641e)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libc.so.6+0x21bf6)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu+0x4964bf)
15:57:00.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
15:57:00.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Could not create new FS layout: Write-ahead log directory /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-2/wal failed to canonicalize: Unable to canonicalize /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304/ts-2: No such file or directory (error 2)
15:57:01.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ThreadSanitizer: reported 4 warnings
15:57:01.348 [DEBUG - main] (MiniKuduCluster.java:195) Response: error {
code: RUNTIME_ERROR
message: "failed to start tablet server 3: /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu: process exited with non-zero status 66"
}
15:57:01.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:57:01.349026 30486 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu with pid 30775
15:57:01.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:57:01.374444 30486 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu with pid 30935
15:57:01.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:57:01.399138 30486 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu with pid 30498
15:57:01.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:57:01.429080 30486 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu with pid 30569
15:57:01.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 15:57:01.453094 30486 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu with pid 30646
15:57:01.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:57:01.477084 30486 env_posix.cc:2125] Unable to access file /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304 during walk: No such file or directory
15:57:01.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:57:01.477304 30486 env_posix.cc:2127] Unable to access file /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304 during walk: No such file or directory
15:57:01.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 15:57:01.477676 30486 tool_action_test.cc:443] Could not delete cluster root: IO error: /tmp/dist-test-taskhTbTJZ/test-tmp/mini-kudu-cluster4025827239515118304: One or more errors occurred
15:57:01.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==30486==WARNING: invalid path to external symbolizer!
15:57:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==30486==WARNING: Failed to use and restart external symbolizer!
15:57:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
15:57:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=30486)
15:57:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 4 at 0x7b6800004200 by main thread (mutexes: write M3882):
15:57:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1becb4)
15:57:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x18e0c2)
15:57:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libmini_oidc.so+0x21aa2)
15:57:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x219ea)
15:57:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0xb3ebe)
15:57:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libmini_cluster.so+0x9918d)
15:57:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libmini_cluster.so+0x9812b)
15:57:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libmini_cluster.so+0x846ba)
15:57:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libmini_cluster.so+0x84819)
15:57:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x656d47)
15:57:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x6531bd)
15:57:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x65288b)
15:57:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x64fac0)
15:57:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x529a01)
15:57:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x52996c)
15:57:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x5298dc)
15:57:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x528545)
15:57:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libkudu_tools_util.so+0x13df8f)
15:57:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libkudu_tools_util.so+0x1307f1)
15:57:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (libkudu_tools_util.so+0x12e582)
15:57:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (kudu+0x6455a0)
15:57:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #21 <null> <null> (kudu+0x645ed7)
15:57:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #22 <null> <null> (kudu+0x64641e)
15:57:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #23 <null> <null> (libc.so.6+0x21bf6)
15:57:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous read of size 4 at 0x7b6800004200 by thread T5:
15:57:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1c0927)
15:57:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (kudu+0x4689ff)
15:57:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) As if synchronized via sleep:
15:57:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x48d9e2)
15:57:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libgutil.so+0xb5f1a)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libkudu_util.so+0x3dd086)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_cluster.so+0x90b3e)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x94017)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libmini_cluster.so+0x88a5f)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libmini_cluster.so+0x86c9a)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x6510a9)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x64f9a4)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x529a01)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x52996c)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x5298dc)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x528545)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x13df8f)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x1307f1)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (libkudu_tools_util.so+0x12e582)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x6455a0)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x645ed7)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (kudu+0x64641e)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (libc.so.6+0x21bf6)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Location is heap block of size 1368 at 0x7b6800004200 allocated by main thread:
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x498c1c)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1bf1c4)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18f020)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x22183)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x857d4)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x6510a9)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x64f9a4)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x529a01)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x52996c)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x5298dc)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x528545)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13df8f)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x1307f1)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x12e582)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x6455a0)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x645ed7)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x64641e)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mutex M3882 (0x7b6800004378) created at:
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x46ca22)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1bff41)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18f020)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x22183)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x857d4)
15:57:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x6510a9)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x64f9a4)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x529a01)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x52996c)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x5298dc)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x528545)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13df8f)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x1307f1)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x12e582)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x6455a0)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x645ed7)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x64641e)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T5 'sq_acceptor' (tid=30493, running) created by main thread at:
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x46c4b5)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c0477)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18f020)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x22183)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x857d4)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x6510a9)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x64f9a4)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x529a01)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x52996c)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x5298dc)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x528545)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13df8f)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x1307f1)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x12e582)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x6455a0)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x645ed7)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x64641e)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-taskhTbTJZ/build/tsan/lib/libserver_process.so+0x1becb4)
15:57:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=30486)
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 4 at 0x7b6800003c00 by main thread (mutexes: write M3879):
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1becb4)
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x18e0c2)
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libmini_oidc.so+0x21ac5)
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x219ea)
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0xb3ebe)
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libmini_cluster.so+0x9918d)
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libmini_cluster.so+0x9812b)
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libmini_cluster.so+0x846ba)
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libmini_cluster.so+0x84819)
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x656d47)
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x6531bd)
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x65288b)
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x64fac0)
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x529a01)
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x52996c)
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x5298dc)
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x528545)
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libkudu_tools_util.so+0x13df8f)
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libkudu_tools_util.so+0x1307f1)
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (libkudu_tools_util.so+0x12e582)
15:57:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (kudu+0x6455a0)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #21 <null> <null> (kudu+0x645ed7)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #22 <null> <null> (kudu+0x64641e)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #23 <null> <null> (libc.so.6+0x21bf6)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous read of size 4 at 0x7b6800003c00 by thread T4:
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1c0927)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (kudu+0x4689ff)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) As if synchronized via sleep:
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x48d41d)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1bed59)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18e0c2)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x21aa2)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_oidc.so+0x219ea)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libmini_cluster.so+0xb3ebe)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libmini_cluster.so+0x9918d)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libmini_cluster.so+0x9812b)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libmini_cluster.so+0x846ba)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (libmini_cluster.so+0x84819)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x656d47)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x6531bd)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x65288b)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x64fac0)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x529a01)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x52996c)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x5298dc)
15:57:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x528545)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libkudu_tools_util.so+0x13df8f)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (libkudu_tools_util.so+0x1307f1)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (libkudu_tools_util.so+0x12e582)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #21 <null> <null> (kudu+0x6455a0)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #22 <null> <null> (kudu+0x645ed7)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #23 <null> <null> (kudu+0x64641e)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #24 <null> <null> (libc.so.6+0x21bf6)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Location is heap block of size 1368 at 0x7b6800003c00 allocated by main thread:
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x498c1c)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1bf1c4)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18f020)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x21dd0)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x857d4)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x6510a9)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x64f9a4)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x529a01)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x52996c)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x5298dc)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x528545)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13df8f)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x1307f1)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x12e582)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x6455a0)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x645ed7)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x64641e)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mutex M3879 (0x7b6800003d78) created at:
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x46ca22)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1bff41)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18f020)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x21dd0)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x857d4)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x6510a9)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x64f9a4)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x529a01)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x52996c)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x5298dc)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x528545)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13df8f)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x1307f1)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x12e582)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x6455a0)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x645ed7)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x64641e)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T4 'sq_acceptor' (tid=30492, running) created by main thread at:
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x46c4b5)
15:57:01.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c0477)
15:57:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18f020)
15:57:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x21dd0)
15:57:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x857d4)
15:57:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x6510a9)
15:57:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x64f9a4)
15:57:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x529a01)
15:57:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x52996c)
15:57:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x5298dc)
15:57:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x528545)
15:57:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13df8f)
15:57:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x1307f1)
15:57:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x12e582)
15:57:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x6455a0)
15:57:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x645ed7)
15:57:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x64641e)
15:57:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
15:57:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:57:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-taskhTbTJZ/build/tsan/lib/libserver_process.so+0x1becb4)
15:57:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
15:57:01.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-14T15:57:01Z chronyd exiting
15:57:02.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ThreadSanitizer: reported 2 warnings
15:57:02.552 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduClient.testStrings: failed attempt 1
java.io.IOException: failed to start tablet server 3: /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu: process exited with non-zero status 66
at org.apache.kudu.test.cluster.MiniKuduCluster.sendRequestToCluster(MiniKuduCluster.java:199) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster.start(MiniKuduCluster.java:266) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster.access$300(MiniKuduCluster.java:85) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster$MiniKuduClusterBuilder.build(MiniKuduCluster.java:883) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:156) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.io.IOException: failed to start tablet server 3: /tmp/dist-test-taskhTbTJZ/build/tsan/bin/kudu: process exited with non-zero status 66
at org.apache.kudu.test.cluster.MiniKuduCluster.sendRequestToCluster(MiniKuduCluster.java:199)
at org.apache.kudu.test.cluster.MiniKuduCluster.start(MiniKuduCluster.java:266)
at org.apache.kudu.test.cluster.MiniKuduCluster.access$300(MiniKuduCluster.java:85)
at org.apache.kudu.test.cluster.MiniKuduCluster$MiniKuduClusterBuilder.build(MiniKuduCluster.java:883)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:156)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)