02:21:20.837 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
02:21:20.837 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskH7s0MS/test-tmp
02:21:20.837 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:21:20.838 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskH7s0MS/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:21:20.839 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274"
extra_tserver_flags: "--scanner_ttl_ms=1000"
extra_tserver_flags: "--scanner_gc_check_interval_us=500000"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
02:21:21.158 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:21:21.158 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:21:21.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:21.161948 2784 mini_oidc.cc:150] Starting JWKS server
02:21:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:21.165931 2784 webserver.cc:469] Webserver started at http://127.0.0.1:33277/ using document root <none> and password file <none>
02:21:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:21.166595 2784 mini_oidc.cc:186] Starting OIDC Discovery server
02:21:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:21.169078 2784 webserver.cc:469] Webserver started at http://127.0.0.1:35421/ using document root <none> and password file <none>
02:21:21.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-26T02:21:21Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:21:21.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-26T02:21:21Z Disabled control of system clock
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:21.236063 2784 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-0/wal
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-0/data
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-0/logs
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-0/data/info.pb
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.184.62:44755
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.184.62
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.184.20:37367
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:21:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.184.62:44755,127.2.184.61:40497,127.2.184.60:37139
02:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:21:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33277/jwks.json with env {}
02:21:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:21.536594 2796 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:21:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:21.537194 2796 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:21:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:21.537721 2796 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:21:21.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:21.569314 2796 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:21:21.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:21.569608 2796 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:21:21.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:21.569804 2796 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:21:21.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:21.569999 2796 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:21:21.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:21.570186 2796 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:21:21.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:21.570338 2796 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33277/jwks.json
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:21.606050 2796 master_runner.cc:386] Master server non-default flags:
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.184.20:37367
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-0/data
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-0/wal
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.184.62:44755,127.2.184.61:40497,127.2.184.60:37139
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.184.62:44755
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33277/jwks.json
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-0/data/info.pb
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.184.62
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:21:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-0/logs
02:21:21.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:21:21.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:21:21.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:21.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:21:21.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:21:21.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7c956859e2f49c4cf1caffa969c1777a7a5d81c
02:21:21.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:21:21.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Jun 2025 01:43:20 UTC on 5fd53c4cbb9d
02:21:21.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6773
02:21:21.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:21:21.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:21.607369 2796 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:21:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:21.608979 2796 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:21:21.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:21.623729 2803 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:21:22.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:21.626087 2802 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:21:22.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:21.624270 2805 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:21:22.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:22.797845 2804 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:21:22.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.797972 2796 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:21:22.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.801815 2796 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:21:22.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.804383 2796 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:21:22.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.805742 2796 hybrid_clock.cc:648] HybridClock initialized: now 1750904482805693 us; error 55 us; skew 500 ppm
02:21:22.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.806586 2796 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:21:22.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.824522 2796 webserver.cc:469] Webserver started at http://127.2.184.62:36193/ using document root <none> and password file <none>
02:21:22.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.825562 2796 fs_manager.cc:362] Metadata directory not provided
02:21:22.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.825788 2796 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:21:22.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.826305 2796 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
02:21:22.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.830864 2796 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-0/data/instance:
02:21:22.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff32056fb9df4aada3db8d77aa1c03b4"
02:21:22.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-26 02:21:22 on dist-test-slave-t7fr"
02:21:22.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.832001 2796 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-0/wal/instance:
02:21:22.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff32056fb9df4aada3db8d77aa1c03b4"
02:21:22.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-26 02:21:22 on dist-test-slave-t7fr"
02:21:22.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.839207 2796 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
02:21:22.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.845057 2813 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:21:22.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.846146 2796 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
02:21:22.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.846500 2796 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-0/data,/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-0/wal
02:21:22.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff32056fb9df4aada3db8d77aa1c03b4"
02:21:22.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-26 02:21:22 on dist-test-slave-t7fr"
02:21:22.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.846838 2796 fs_report.cc:389] FS layout report
02:21:22.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:21:22.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-0/wal
02:21:22.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-0/wal
02:21:22.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-0/data/data
02:21:22.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:21:22.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:21:22.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:21:22.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:21:22.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:21:22.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:21:22.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:21:22.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:21:22.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:21:22.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:21:22.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:21:22.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.901484 2796 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:21:22.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.902999 2796 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:21:22.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.903497 2796 kserver.cc:163] Server-wide thread pool size limit: 3276
02:21:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.976377 2796 rpc_server.cc:307] RPC server started. Bound to: 127.2.184.62:44755
02:21:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.976467 2864 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.184.62:44755 every 8 connection(s)
02:21:22.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.979116 2796 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-0/data/info.pb
02:21:22.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.981449 2784 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu as pid 2796
02:21:22.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.982010 2784 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-0/wal/instance
02:21:22.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.985559 2865 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:21:22.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.987365 2784 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu
02:21:22.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu
02:21:22.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-1/wal
02:21:22.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-1/data
02:21:22.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:21:22.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:21:22.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:21:22.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:21:22.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:21:22.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:21:22.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-1/logs
02:21:22.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-1/data/info.pb
02:21:22.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:21:22.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:21:22.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:21:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:21:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:21:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:21:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:21:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:21:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:21:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:21:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:21:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:21:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.184.61:40497
02:21:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.184.61
02:21:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:21:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.184.20:37367
02:21:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:21:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:21:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:21:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:21:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.184.62:44755,127.2.184.61:40497,127.2.184.60:37139
02:21:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:21:22.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33277/jwks.json with env {}
02:21:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:22.999017 2865 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.184.62" port: 44755 } has no permanent_uuid. Determining permanent_uuid...
02:21:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:23.023733 2865 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } has no permanent_uuid. Determining permanent_uuid...
02:21:23.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:23.026034 2815 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.184.61:40497: connect: Connection refused (error 111)
02:21:23.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:23.030385 2865 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.184.61:40497: Network error: Client connection negotiation failed: client connection to 127.2.184.61:40497: connect: Connection refused (error 111)
02:21:23.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:23.080015 2865 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } attempt: 1
02:21:23.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:23.084215 2865 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.184.61:40497: Network error: Client connection negotiation failed: client connection to 127.2.184.61:40497: connect: Connection refused (error 111)
02:21:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:23.152731 2865 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } attempt: 2
02:21:23.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:23.156780 2865 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.184.61:40497: Network error: Client connection negotiation failed: client connection to 127.2.184.61:40497: connect: Connection refused (error 111)
02:21:23.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:23.248312 2865 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } attempt: 3
02:21:23.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:23.252188 2865 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.184.61:40497: Network error: Client connection negotiation failed: client connection to 127.2.184.61:40497: connect: Connection refused (error 111)
02:21:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:23.292642 2867 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:21:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:23.293234 2867 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:21:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:23.293717 2867 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:21:23.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:23.324307 2867 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:21:23.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:23.324600 2867 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:21:23.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:23.324798 2867 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:21:23.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:23.325002 2867 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:21:23.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:23.325212 2867 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:21:23.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:23.325357 2867 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33277/jwks.json
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:23.360154 2867 master_runner.cc:386] Master server non-default flags:
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.184.20:37367
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-1/data
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-1/wal
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.184.62:44755,127.2.184.61:40497,127.2.184.60:37139
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.184.61:40497
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33277/jwks.json
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-1/data/info.pb
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.184.61
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:21:23.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:21:23.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:21:23.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:21:23.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-1/logs
02:21:23.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:21:23.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:21:23.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:23.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:21:23.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:21:23.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7c956859e2f49c4cf1caffa969c1777a7a5d81c
02:21:23.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:21:23.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Jun 2025 01:43:20 UTC on 5fd53c4cbb9d
02:21:23.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6773
02:21:23.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:21:23.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:23.361515 2867 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:21:23.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:23.363118 2867 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:21:23.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:23.378783 2879 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:21:23.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:23.395648 2865 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } attempt: 4
02:21:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:23.399519 2865 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.184.61:40497: Network error: Client connection negotiation failed: client connection to 127.2.184.61:40497: connect: Connection refused (error 111)
02:21:23.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:23.699216 2865 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } attempt: 5
02:21:23.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:23.703465 2865 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.184.61:40497: Network error: Client connection negotiation failed: client connection to 127.2.184.61:40497: connect: Connection refused (error 111)
02:21:24.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.250856 2865 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } attempt: 6
02:21:24.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:24.265259 2865 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.184.61:40497: Network error: Client connection negotiation failed: client connection to 127.2.184.61:40497: connect: Connection refused (error 111)
02:21:24.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:23.379212 2877 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:21:24.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:23.379349 2876 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:21:24.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:23.381466 2867 server_base.cc:1048] running on GCE node
02:21:24.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.568117 2867 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:21:24.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.571282 2867 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:21:24.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.572710 2867 hybrid_clock.cc:648] HybridClock initialized: now 1750904484572668 us; error 60 us; skew 500 ppm
02:21:24.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.573539 2867 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:21:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.580327 2867 webserver.cc:469] Webserver started at http://127.2.184.61:36417/ using document root <none> and password file <none>
02:21:24.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.581302 2867 fs_manager.cc:362] Metadata directory not provided
02:21:24.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.581493 2867 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:21:24.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.581892 2867 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
02:21:24.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.586234 2867 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-1/data/instance:
02:21:24.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7"
02:21:24.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-26 02:21:24 on dist-test-slave-t7fr"
02:21:24.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.587265 2867 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-1/wal/instance:
02:21:24.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7"
02:21:24.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-26 02:21:24 on dist-test-slave-t7fr"
02:21:24.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.594393 2867 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
02:21:24.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.599771 2889 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:21:24.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.600785 2867 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
02:21:24.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.601116 2867 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-1/data,/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-1/wal
02:21:24.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7"
02:21:24.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-26 02:21:24 on dist-test-slave-t7fr"
02:21:24.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.601480 2867 fs_report.cc:389] FS layout report
02:21:24.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:21:24.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-1/wal
02:21:24.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-1/wal
02:21:24.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-1/data/data
02:21:24.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:21:24.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:21:24.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:21:24.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:21:24.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:21:24.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:21:24.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:21:24.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:21:24.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:21:24.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:21:24.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:21:24.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.662504 2867 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:21:24.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.664002 2867 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:21:24.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.664479 2867 kserver.cc:163] Server-wide thread pool size limit: 3276
02:21:24.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.735221 2867 rpc_server.cc:307] RPC server started. Bound to: 127.2.184.61:40497
02:21:24.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.735311 2940 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.184.61:40497 every 8 connection(s)
02:21:24.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.737871 2867 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-1/data/info.pb
02:21:24.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.742931 2941 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:21:24.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.743770 2784 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu as pid 2867
02:21:24.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.744113 2784 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-1/wal/instance
02:21:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.746093 2784 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu
02:21:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu
02:21:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-2/wal
02:21:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-2/data
02:21:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:21:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:21:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:21:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:21:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:21:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:21:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-2/logs
02:21:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-2/data/info.pb
02:21:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:21:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:21:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:21:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:21:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:21:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:21:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:21:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:21:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:21:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:21:24.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:21:24.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:21:24.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.184.60:37139
02:21:24.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.184.60
02:21:24.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:21:24.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.184.20:37367
02:21:24.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:21:24.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:21:24.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:21:24.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:21:24.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.184.62:44755,127.2.184.61:40497,127.2.184.60:37139
02:21:24.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:21:24.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33277/jwks.json with env {}
02:21:24.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.756004 2941 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.184.62" port: 44755 } has no permanent_uuid. Determining permanent_uuid...
02:21:24.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.779481 2941 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } has no permanent_uuid. Determining permanent_uuid...
02:21:24.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.793905 2941 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } has no permanent_uuid. Determining permanent_uuid...
02:21:24.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:24.796494 2893 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.184.60:37139: connect: Connection refused (error 111)
02:21:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:24.800823 2941 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.184.60:37139: Network error: Client connection negotiation failed: client connection to 127.2.184.60:37139: connect: Connection refused (error 111)
02:21:24.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.850297 2941 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } attempt: 1
02:21:24.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:24.854729 2941 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.184.60:37139: Network error: Client connection negotiation failed: client connection to 127.2.184.60:37139: connect: Connection refused (error 111)
02:21:24.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:24.923213 2941 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } attempt: 2
02:21:24.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:24.927347 2941 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.184.60:37139: Network error: Client connection negotiation failed: client connection to 127.2.184.60:37139: connect: Connection refused (error 111)
02:21:25.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:25.018815 2941 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } attempt: 3
02:21:25.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:25.022817 2941 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.184.60:37139: Network error: Client connection negotiation failed: client connection to 127.2.184.60:37139: connect: Connection refused (error 111)
02:21:25.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:25.070384 2943 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:21:25.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:25.071007 2943 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:21:25.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:25.071570 2943 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:21:25.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:25.102864 2943 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:21:25.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:25.103241 2943 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:21:25.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:25.103569 2943 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:21:25.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:25.103835 2943 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:21:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:25.104135 2943 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:21:25.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:25.104355 2943 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33277/jwks.json
02:21:25.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:25.139328 2943 master_runner.cc:386] Master server non-default flags:
02:21:25.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:21:25.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.184.20:37367
02:21:25.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:21:25.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:21:25.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-2/data
02:21:25.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-2/wal
02:21:25.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.184.62:44755,127.2.184.61:40497,127.2.184.60:37139
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.184.60:37139
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33277/jwks.json
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-2/data/info.pb
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.184.60
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-2/logs
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7c956859e2f49c4cf1caffa969c1777a7a5d81c
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Jun 2025 01:43:20 UTC on 5fd53c4cbb9d
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6773
02:21:25.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:21:25.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:25.141005 2943 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:21:25.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:25.142684 2943 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:21:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:25.158115 2954 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:21:25.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:25.166198 2941 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } attempt: 4
02:21:25.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:25.171988 2941 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.184.60:37139: Network error: Client connection negotiation failed: client connection to 127.2.184.60:37139: connect: Connection refused (error 111)
02:21:25.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:25.325927 2865 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } attempt: 7
02:21:25.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:25.337847 2865 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } has no permanent_uuid. Determining permanent_uuid...
02:21:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:25.344133 2865 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.184.60:37139: Network error: Client connection negotiation failed: client connection to 127.2.184.60:37139: connect: Connection refused (error 111)
02:21:25.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:25.402464 2865 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } attempt: 1
02:21:25.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:25.406183 2865 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.184.60:37139: Network error: Client connection negotiation failed: client connection to 127.2.184.60:37139: connect: Connection refused (error 111)
02:21:25.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:25.471383 2941 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } attempt: 5
02:21:25.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:25.476466 2941 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.184.60:37139: Network error: Client connection negotiation failed: client connection to 127.2.184.60:37139: connect: Connection refused (error 111)
02:21:25.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:25.487840 2865 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } attempt: 2
02:21:25.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:25.491876 2865 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.184.60:37139: Network error: Client connection negotiation failed: client connection to 127.2.184.60:37139: connect: Connection refused (error 111)
02:21:25.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:25.577493 2865 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } attempt: 3
02:21:25.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:25.583284 2865 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.184.60:37139: Network error: Client connection negotiation failed: client connection to 127.2.184.60:37139: connect: Connection refused (error 111)
02:21:25.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:25.723722 2865 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } attempt: 4
02:21:25.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:25.729241 2865 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.184.60:37139: Network error: Client connection negotiation failed: client connection to 127.2.184.60:37139: connect: Connection refused (error 111)
02:21:26.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.012692 2865 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } attempt: 5
02:21:26.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:26.018112 2865 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.184.60:37139: Network error: Client connection negotiation failed: client connection to 127.2.184.60:37139: connect: Connection refused (error 111)
02:21:26.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.023866 2941 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } attempt: 6
02:21:26.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:26.030753 2941 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.184.60:37139: Network error: Client connection negotiation failed: client connection to 127.2.184.60:37139: connect: Connection refused (error 111)
02:21:26.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:25.159255 2956 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:21:26.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:25.158346 2953 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:21:26.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:26.351814 2955 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:21:26.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.352238 2943 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:21:26.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.355916 2943 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:21:26.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.358595 2943 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:21:26.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.359951 2943 hybrid_clock.cc:648] HybridClock initialized: now 1750904486359926 us; error 49 us; skew 500 ppm
02:21:26.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.360828 2943 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:21:26.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.368008 2943 webserver.cc:469] Webserver started at http://127.2.184.60:38419/ using document root <none> and password file <none>
02:21:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.368932 2943 fs_manager.cc:362] Metadata directory not provided
02:21:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.369124 2943 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:21:26.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.369632 2943 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
02:21:26.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.374264 2943 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-2/data/instance:
02:21:26.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b2763111901d4eff91ab314295dd02a0"
02:21:26.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-26 02:21:26 on dist-test-slave-t7fr"
02:21:26.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.375382 2943 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-2/wal/instance:
02:21:26.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b2763111901d4eff91ab314295dd02a0"
02:21:26.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-26 02:21:26 on dist-test-slave-t7fr"
02:21:26.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.383006 2943 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.003s
02:21:26.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.388760 2968 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:21:26.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.389909 2943 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
02:21:26.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.390221 2943 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-2/data,/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-2/wal
02:21:26.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b2763111901d4eff91ab314295dd02a0"
02:21:26.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-26 02:21:26 on dist-test-slave-t7fr"
02:21:26.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.390547 2943 fs_report.cc:389] FS layout report
02:21:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:21:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-2/wal
02:21:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-2/wal
02:21:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-2/data/data
02:21:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:21:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:21:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:21:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:21:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:21:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:21:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:21:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:21:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:21:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:21:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:21:26.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.448442 2943 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:21:26.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.450028 2943 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:21:26.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.450452 2943 kserver.cc:163] Server-wide thread pool size limit: 3276
02:21:26.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.524371 2943 rpc_server.cc:307] RPC server started. Bound to: 127.2.184.60:37139
02:21:26.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.524488 3019 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.184.60:37139 every 8 connection(s)
02:21:26.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.527189 2943 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-2/data/info.pb
02:21:26.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.532608 3020 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:21:26.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.534864 2784 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu as pid 2943
02:21:26.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.535385 2784 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/master-2/wal/instance
02:21:26.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.549755 3020 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.184.62" port: 44755 } has no permanent_uuid. Determining permanent_uuid...
02:21:26.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.570529 2865 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } attempt: 6
02:21:26.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.581945 3020 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } has no permanent_uuid. Determining permanent_uuid...
02:21:26.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.594189 3020 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } has no permanent_uuid. Determining permanent_uuid...
02:21:26.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.601970 2865 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4: Bootstrap starting.
02:21:26.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.608306 2865 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4: Neither blocks nor log segments found. Creating new log.
02:21:26.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.610636 2865 log.cc:826] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4: Log is configured to *not* fsync() on all Append() calls
02:21:26.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.612236 3020 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0: Bootstrap starting.
02:21:26.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.615504 2865 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4: No bootstrap required, opened a new log
02:21:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.618249 3020 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0: Neither blocks nor log segments found. Creating new log.
02:21:26.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.620549 3020 log.cc:826] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0: Log is configured to *not* fsync() on all Append() calls
02:21:26.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.625766 3020 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0: No bootstrap required, opened a new log
02:21:26.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.637365 2865 raft_consensus.cc:357] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff32056fb9df4aada3db8d77aa1c03b4" member_type: VOTER last_known_addr { host: "127.2.184.62" port: 44755 } } peers { permanent_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7" member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } } peers { permanent_uuid: "b2763111901d4eff91ab314295dd02a0" member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } }
02:21:26.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.638159 2865 raft_consensus.cc:383] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:21:26.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.638373 2865 raft_consensus.cc:738] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ff32056fb9df4aada3db8d77aa1c03b4, State: Initialized, Role: FOLLOWER
02:21:26.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.639147 2865 consensus_queue.cc:260] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff32056fb9df4aada3db8d77aa1c03b4" member_type: VOTER last_known_addr { host: "127.2.184.62" port: 44755 } } peers { permanent_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7" member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } } peers { permanent_uuid: "b2763111901d4eff91ab314295dd02a0" member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } }
02:21:26.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.642382 3033 sys_catalog.cc:455] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff32056fb9df4aada3db8d77aa1c03b4" member_type: VOTER last_known_addr { host: "127.2.184.62" port: 44755 } } peers { permanent_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7" member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } } peers { permanent_uuid: "b2763111901d4eff91ab314295dd02a0" member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } } }
02:21:26.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.643414 3033 sys_catalog.cc:458] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4 [sys.catalog]: This master's current role is: FOLLOWER
02:21:26.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.644381 2865 sys_catalog.cc:564] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4 [sys.catalog]: configured and running, proceeding with master startup.
02:21:26.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.647184 3020 raft_consensus.cc:357] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff32056fb9df4aada3db8d77aa1c03b4" member_type: VOTER last_known_addr { host: "127.2.184.62" port: 44755 } } peers { permanent_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7" member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } } peers { permanent_uuid: "b2763111901d4eff91ab314295dd02a0" member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } }
02:21:26.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.647868 3020 raft_consensus.cc:383] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:21:26.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.648171 3020 raft_consensus.cc:738] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b2763111901d4eff91ab314295dd02a0, State: Initialized, Role: FOLLOWER
02:21:26.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.648953 3020 consensus_queue.cc:260] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff32056fb9df4aada3db8d77aa1c03b4" member_type: VOTER last_known_addr { host: "127.2.184.62" port: 44755 } } peers { permanent_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7" member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } } peers { permanent_uuid: "b2763111901d4eff91ab314295dd02a0" member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } }
02:21:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.654963 3037 sys_catalog.cc:455] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff32056fb9df4aada3db8d77aa1c03b4" member_type: VOTER last_known_addr { host: "127.2.184.62" port: 44755 } } peers { permanent_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7" member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } } peers { permanent_uuid: "b2763111901d4eff91ab314295dd02a0" member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } } }
02:21:26.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.656150 3037 sys_catalog.cc:458] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [sys.catalog]: This master's current role is: FOLLOWER
02:21:26.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.657004 3020 sys_catalog.cc:564] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [sys.catalog]: configured and running, proceeding with master startup.
02:21:26.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:26.662467 2819 tablet.cc:2378] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:21:26.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:26.679525 3050 catalog_manager.cc:1560] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:21:26.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:26.679870 3050 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:21:26.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:26.705788 2974 tablet.cc:2378] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:21:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.679090 2796 master_runner.cc:186] Error getting master registration for 127.2.184.61:40497: OK, instance_id { permanent_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7" instance_seqno: 1750904484716090 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:21:26.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.720613 2796 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:21:26.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:26.724457 3057 catalog_manager.cc:1560] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:21:26.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:26.724812 3057 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:21:26.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.702787 2943 master_runner.cc:186] Error getting master registration for 127.2.184.61:40497: OK, instance_id { permanent_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7" instance_seqno: 1750904484716090 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:21:26.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.731536 2943 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:21:26.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.889607 3037 raft_consensus.cc:491] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:21:26.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.890177 3037 raft_consensus.cc:513] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff32056fb9df4aada3db8d77aa1c03b4" member_type: VOTER last_known_addr { host: "127.2.184.62" port: 44755 } } peers { permanent_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7" member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } } peers { permanent_uuid: "b2763111901d4eff91ab314295dd02a0" member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } }
02:21:26.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.892572 3037 leader_election.cc:290] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ff32056fb9df4aada3db8d77aa1c03b4 (127.2.184.62:44755), ee82c1cc1a144d0bb22efcf4694b3ae7 (127.2.184.61:40497)
02:21:26.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.894418 2840 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b2763111901d4eff91ab314295dd02a0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ff32056fb9df4aada3db8d77aa1c03b4" is_pre_election: true
02:21:26.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.895262 2840 raft_consensus.cc:2466] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b2763111901d4eff91ab314295dd02a0 in term 0.
02:21:26.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.896864 2970 leader_election.cc:304] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: b2763111901d4eff91ab314295dd02a0, ff32056fb9df4aada3db8d77aa1c03b4; no voters:
02:21:26.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.898072 3037 raft_consensus.cc:2802] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [term 0 FOLLOWER]: Leader pre-election won for term 1
02:21:26.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.898526 3037 raft_consensus.cc:491] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:21:26.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.900592 3037 raft_consensus.cc:3058] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [term 0 FOLLOWER]: Advancing to term 1
02:21:26.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.907810 3037 raft_consensus.cc:513] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff32056fb9df4aada3db8d77aa1c03b4" member_type: VOTER last_known_addr { host: "127.2.184.62" port: 44755 } } peers { permanent_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7" member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } } peers { permanent_uuid: "b2763111901d4eff91ab314295dd02a0" member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } }
02:21:26.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.910765 2840 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b2763111901d4eff91ab314295dd02a0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ff32056fb9df4aada3db8d77aa1c03b4"
02:21:26.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.911331 2840 raft_consensus.cc:3058] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4 [term 0 FOLLOWER]: Advancing to term 1
02:21:26.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.918303 2840 raft_consensus.cc:2466] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b2763111901d4eff91ab314295dd02a0 in term 1.
02:21:26.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.919517 2970 leader_election.cc:304] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: b2763111901d4eff91ab314295dd02a0, ff32056fb9df4aada3db8d77aa1c03b4; no voters:
02:21:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.925500 3037 leader_election.cc:290] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [CANDIDATE]: Term 1 election: Requested vote from peers ff32056fb9df4aada3db8d77aa1c03b4 (127.2.184.62:44755), ee82c1cc1a144d0bb22efcf4694b3ae7 (127.2.184.61:40497)
02:21:26.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.926028 3037 raft_consensus.cc:2802] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [term 1 FOLLOWER]: Leader election won for term 1
02:21:26.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.926476 3037 raft_consensus.cc:695] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [term 1 LEADER]: Becoming Leader. State: Replica: b2763111901d4eff91ab314295dd02a0, State: Running, Role: LEADER
02:21:26.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.927373 3037 consensus_queue.cc:237] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff32056fb9df4aada3db8d77aa1c03b4" member_type: VOTER last_known_addr { host: "127.2.184.62" port: 44755 } } peers { permanent_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7" member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } } peers { permanent_uuid: "b2763111901d4eff91ab314295dd02a0" member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } }
02:21:26.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.936637 3059 sys_catalog.cc:455] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b2763111901d4eff91ab314295dd02a0. Latest consensus state: current_term: 1 leader_uuid: "b2763111901d4eff91ab314295dd02a0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff32056fb9df4aada3db8d77aa1c03b4" member_type: VOTER last_known_addr { host: "127.2.184.62" port: 44755 } } peers { permanent_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7" member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } } peers { permanent_uuid: "b2763111901d4eff91ab314295dd02a0" member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } } }
02:21:26.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.937415 3059 sys_catalog.cc:458] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [sys.catalog]: This master's current role is: LEADER
02:21:26.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.949877 3061 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
02:21:26.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.957302 3061 catalog_manager.cc:1486] Initializing Kudu cluster ID...
02:21:26.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.980351 2840 raft_consensus.cc:1273] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4 [term 1 FOLLOWER]: Refusing update from remote peer b2763111901d4eff91ab314295dd02a0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:21:26.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:26.982383 3059 consensus_queue.cc:1035] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ff32056fb9df4aada3db8d77aa1c03b4" member_type: VOTER last_known_addr { host: "127.2.184.62" port: 44755 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:21:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.025489 3062 mvcc.cc:204] Tried to move back new op lower bound from 7171704778645639168 to 7171704778471112704. Current Snapshot: MvccSnapshot[applied={T|T < 7171704778645639168}]
02:21:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.025954 3033 sys_catalog.cc:455] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b2763111901d4eff91ab314295dd02a0. Latest consensus state: current_term: 1 leader_uuid: "b2763111901d4eff91ab314295dd02a0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff32056fb9df4aada3db8d77aa1c03b4" member_type: VOTER last_known_addr { host: "127.2.184.62" port: 44755 } } peers { permanent_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7" member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } } peers { permanent_uuid: "b2763111901d4eff91ab314295dd02a0" member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } } }
02:21:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.026679 3033 sys_catalog.cc:458] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4 [sys.catalog]: This master's current role is: FOLLOWER
02:21:27.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.044142 3037 sys_catalog.cc:455] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "b2763111901d4eff91ab314295dd02a0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff32056fb9df4aada3db8d77aa1c03b4" member_type: VOTER last_known_addr { host: "127.2.184.62" port: 44755 } } peers { permanent_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7" member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } } peers { permanent_uuid: "b2763111901d4eff91ab314295dd02a0" member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } } }
02:21:27.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.044869 3037 sys_catalog.cc:458] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [sys.catalog]: This master's current role is: LEADER
02:21:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.049222 3033 sys_catalog.cc:455] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "b2763111901d4eff91ab314295dd02a0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff32056fb9df4aada3db8d77aa1c03b4" member_type: VOTER last_known_addr { host: "127.2.184.62" port: 44755 } } peers { permanent_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7" member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } } peers { permanent_uuid: "b2763111901d4eff91ab314295dd02a0" member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } } }
02:21:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.049932 3033 sys_catalog.cc:458] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4 [sys.catalog]: This master's current role is: FOLLOWER
02:21:27.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.053997 3061 catalog_manager.cc:1349] Generated new cluster ID: e8707da6cd0f448ca4a449d4f463cdad
02:21:27.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.054315 3061 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
02:21:27.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.091136 2941 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } attempt: 7
02:21:27.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.100697 3061 catalog_manager.cc:1372] Generated new certificate authority record
02:21:27.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.102870 3061 catalog_manager.cc:1506] Loading token signing keys...
02:21:27.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.143234 2941 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ee82c1cc1a144d0bb22efcf4694b3ae7: Bootstrap starting.
02:21:27.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.162685 2941 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ee82c1cc1a144d0bb22efcf4694b3ae7: Neither blocks nor log segments found. Creating new log.
02:21:27.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.169534 2941 log.cc:826] T 00000000000000000000000000000000 P ee82c1cc1a144d0bb22efcf4694b3ae7: Log is configured to *not* fsync() on all Append() calls
02:21:27.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.179124 3061 catalog_manager.cc:5955] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0: Generated new TSK 0
02:21:27.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.180284 3061 catalog_manager.cc:1516] Initializing in-progress tserver states...
02:21:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.197361 2941 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ee82c1cc1a144d0bb22efcf4694b3ae7: No bootstrap required, opened a new log
02:21:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.224445 2941 raft_consensus.cc:357] T 00000000000000000000000000000000 P ee82c1cc1a144d0bb22efcf4694b3ae7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff32056fb9df4aada3db8d77aa1c03b4" member_type: VOTER last_known_addr { host: "127.2.184.62" port: 44755 } } peers { permanent_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7" member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } } peers { permanent_uuid: "b2763111901d4eff91ab314295dd02a0" member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } }
02:21:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.225667 2941 raft_consensus.cc:383] T 00000000000000000000000000000000 P ee82c1cc1a144d0bb22efcf4694b3ae7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:21:27.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.226063 2941 raft_consensus.cc:738] T 00000000000000000000000000000000 P ee82c1cc1a144d0bb22efcf4694b3ae7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ee82c1cc1a144d0bb22efcf4694b3ae7, State: Initialized, Role: FOLLOWER
02:21:27.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.227035 2941 consensus_queue.cc:260] T 00000000000000000000000000000000 P ee82c1cc1a144d0bb22efcf4694b3ae7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff32056fb9df4aada3db8d77aa1c03b4" member_type: VOTER last_known_addr { host: "127.2.184.62" port: 44755 } } peers { permanent_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7" member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } } peers { permanent_uuid: "b2763111901d4eff91ab314295dd02a0" member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } }
02:21:27.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.233345 2941 sys_catalog.cc:564] T 00000000000000000000000000000000 P ee82c1cc1a144d0bb22efcf4694b3ae7 [sys.catalog]: configured and running, proceeding with master startup.
02:21:27.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.233291 2784 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu
02:21:27.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu
02:21:27.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-0/wal
02:21:27.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-0/data
02:21:27.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:21:27.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:21:27.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:21:27.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:21:27.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:21:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:21:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-0/logs
02:21:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-0/data/info.pb
02:21:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:21:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:21:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:21:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:21:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:21:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:21:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:21:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:21:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:21:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:21:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.184.1:0
02:21:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.184.1
02:21:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.184.1
02:21:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:21:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.184.62:44755,127.2.184.61:40497,127.2.184.60:37139
02:21:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.184.20:37367
02:21:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:21:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:21:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:21:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --scanner_ttl_ms=1000
02:21:27.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --scanner_gc_check_interval_us=500000 with env {}
02:21:27.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.233899 2914 raft_consensus.cc:3058] T 00000000000000000000000000000000 P ee82c1cc1a144d0bb22efcf4694b3ae7 [term 0 FOLLOWER]: Advancing to term 1
02:21:27.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.235355 2915 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b2763111901d4eff91ab314295dd02a0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7"
02:21:27.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.234416 2916 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b2763111901d4eff91ab314295dd02a0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7" is_pre_election: true
02:21:27.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.232314 3069 sys_catalog.cc:455] T 00000000000000000000000000000000 P ee82c1cc1a144d0bb22efcf4694b3ae7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff32056fb9df4aada3db8d77aa1c03b4" member_type: VOTER last_known_addr { host: "127.2.184.62" port: 44755 } } peers { permanent_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7" member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } } peers { permanent_uuid: "b2763111901d4eff91ab314295dd02a0" member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } } }
02:21:27.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.239796 3069 sys_catalog.cc:458] T 00000000000000000000000000000000 P ee82c1cc1a144d0bb22efcf4694b3ae7 [sys.catalog]: This master's current role is: FOLLOWER
02:21:27.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.242588 2914 raft_consensus.cc:1273] T 00000000000000000000000000000000 P ee82c1cc1a144d0bb22efcf4694b3ae7 [term 1 FOLLOWER]: Refusing update from remote peer b2763111901d4eff91ab314295dd02a0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:21:27.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.244299 3059 consensus_queue.cc:1035] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7" member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:21:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.308368 3069 sys_catalog.cc:455] T 00000000000000000000000000000000 P ee82c1cc1a144d0bb22efcf4694b3ae7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b2763111901d4eff91ab314295dd02a0. Latest consensus state: current_term: 1 leader_uuid: "b2763111901d4eff91ab314295dd02a0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff32056fb9df4aada3db8d77aa1c03b4" member_type: VOTER last_known_addr { host: "127.2.184.62" port: 44755 } } peers { permanent_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7" member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } } peers { permanent_uuid: "b2763111901d4eff91ab314295dd02a0" member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } } }
02:21:27.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.309414 3069 sys_catalog.cc:458] T 00000000000000000000000000000000 P ee82c1cc1a144d0bb22efcf4694b3ae7 [sys.catalog]: This master's current role is: FOLLOWER
02:21:27.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.311385 3075 sys_catalog.cc:455] T 00000000000000000000000000000000 P ee82c1cc1a144d0bb22efcf4694b3ae7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "b2763111901d4eff91ab314295dd02a0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff32056fb9df4aada3db8d77aa1c03b4" member_type: VOTER last_known_addr { host: "127.2.184.62" port: 44755 } } peers { permanent_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7" member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } } peers { permanent_uuid: "b2763111901d4eff91ab314295dd02a0" member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } } }
02:21:27.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.312286 3075 sys_catalog.cc:458] T 00000000000000000000000000000000 P ee82c1cc1a144d0bb22efcf4694b3ae7 [sys.catalog]: This master's current role is: FOLLOWER
02:21:27.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.388989 3087 catalog_manager.cc:1261] Loaded cluster ID: e8707da6cd0f448ca4a449d4f463cdad
02:21:27.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.389312 3087 catalog_manager.cc:1554] T 00000000000000000000000000000000 P ee82c1cc1a144d0bb22efcf4694b3ae7: loading cluster ID for follower catalog manager: success
02:21:27.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.401983 3087 catalog_manager.cc:1576] T 00000000000000000000000000000000 P ee82c1cc1a144d0bb22efcf4694b3ae7: acquiring CA information for follower catalog manager: success
02:21:27.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.407660 3087 catalog_manager.cc:1604] T 00000000000000000000000000000000 P ee82c1cc1a144d0bb22efcf4694b3ae7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:21:27.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.644063 3059 sys_catalog.cc:455] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "b2763111901d4eff91ab314295dd02a0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ff32056fb9df4aada3db8d77aa1c03b4" member_type: VOTER last_known_addr { host: "127.2.184.62" port: 44755 } } peers { permanent_uuid: "ee82c1cc1a144d0bb22efcf4694b3ae7" member_type: VOTER last_known_addr { host: "127.2.184.61" port: 40497 } } peers { permanent_uuid: "b2763111901d4eff91ab314295dd02a0" member_type: VOTER last_known_addr { host: "127.2.184.60" port: 37139 } } }
02:21:27.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.644800 3059 sys_catalog.cc:458] T 00000000000000000000000000000000 P b2763111901d4eff91ab314295dd02a0 [sys.catalog]: This master's current role is: LEADER
02:21:27.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.685806 3050 catalog_manager.cc:1261] Loaded cluster ID: e8707da6cd0f448ca4a449d4f463cdad
02:21:27.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.686296 3050 catalog_manager.cc:1554] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4: loading cluster ID for follower catalog manager: success
02:21:27.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.693393 3050 catalog_manager.cc:1576] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4: acquiring CA information for follower catalog manager: success
02:21:27.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.704476 3050 catalog_manager.cc:1604] T 00000000000000000000000000000000 P ff32056fb9df4aada3db8d77aa1c03b4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:21:27.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:27.821769 3070 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:21:27.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:27.825124 3070 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:21:27.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:27.828967 3070 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:21:27.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:27.916869 3070 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:21:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:27.919627 3070 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.184.1
02:21:27.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.985496 3070 tablet_server_runner.cc:78] Tablet server non-default flags:
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.184.20:37367
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-0/data
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-0/wal
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.184.1:0
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-0/data/info.pb
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.184.1
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --scanner_gc_check_interval_us=500000
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --scanner_ttl_ms=1000
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.184.62:44755,127.2.184.61:40497,127.2.184.60:37139
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.184.1
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-0/logs
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:27.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:21:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:21:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7c956859e2f49c4cf1caffa969c1777a7a5d81c
02:21:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:21:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Jun 2025 01:43:20 UTC on 5fd53c4cbb9d
02:21:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6773
02:21:27.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:21:27.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.988561 3070 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:21:27.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:27.991124 3070 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:21:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:28.047698 3096 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:21:30.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:29.433210 3094 debug-util.cc:398] Leaking SignalData structure 0x7b0800015620 after lost signal to thread 3070
02:21:30.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:30.399334 3094 kernel_stack_watchdog.cc:198] Thread 3070 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:641 for 381ms:
02:21:30.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:21:30.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:21:30.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:21:30.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:21:30.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:28.052284 3095 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:21:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:30.400991 3070 thread.cc:640] OpenStack (cloud detector) Time spent creating pthread: real 2.352s user 0.689s sys 1.275s
02:21:30.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:30.401485 3098 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:21:30.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:30.401499 3070 thread.cc:606] OpenStack (cloud detector) Time spent starting thread: real 2.352s user 0.689s sys 1.275s
02:21:30.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:30.404579 3097 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2353 milliseconds
02:21:30.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.404598 3070 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:21:30.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.405949 3070 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:21:30.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.408562 3070 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:21:30.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.410010 3070 hybrid_clock.cc:648] HybridClock initialized: now 1750904490409923 us; error 93 us; skew 500 ppm
02:21:30.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.411098 3070 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:21:30.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.422506 3070 webserver.cc:469] Webserver started at http://127.2.184.1:42331/ using document root <none> and password file <none>
02:21:30.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.423418 3070 fs_manager.cc:362] Metadata directory not provided
02:21:30.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.423614 3070 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:21:30.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.424083 3070 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
02:21:30.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.428633 3070 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-0/data/instance:
02:21:30.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4bcdd8293835450d8de2c1e14364a517"
02:21:30.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-26 02:21:30 on dist-test-slave-t7fr"
02:21:30.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.429728 3070 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-0/wal/instance:
02:21:30.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4bcdd8293835450d8de2c1e14364a517"
02:21:30.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-26 02:21:30 on dist-test-slave-t7fr"
02:21:30.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.437454 3070 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.002s
02:21:30.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.443445 3105 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:21:30.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.444526 3070 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
02:21:30.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.444811 3070 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-0/data,/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-0/wal
02:21:30.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4bcdd8293835450d8de2c1e14364a517"
02:21:30.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-26 02:21:30 on dist-test-slave-t7fr"
02:21:30.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.445112 3070 fs_report.cc:389] FS layout report
02:21:30.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:21:30.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-0/wal
02:21:30.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-0/wal
02:21:30.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-0/data/data
02:21:30.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:21:30.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:21:30.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:21:30.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:21:30.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:21:30.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:21:30.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:21:30.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:21:30.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:21:30.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:21:30.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:21:30.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.500289 3070 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:21:30.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.501840 3070 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:21:30.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.502314 3070 kserver.cc:163] Server-wide thread pool size limit: 3276
02:21:30.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.505652 3070 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:21:30.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.510447 3070 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:21:30.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.510668 3070 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:21:30.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.510972 3070 ts_tablet_manager.cc:610] Registered 0 tablets
02:21:30.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.511169 3070 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:21:30.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.691956 3070 rpc_server.cc:307] RPC server started. Bound to: 127.2.184.1:40481
02:21:30.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.692137 3218 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.184.1:40481 every 8 connection(s)
02:21:30.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.695109 3070 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-0/data/info.pb
02:21:30.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.704185 2784 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu as pid 3070
02:21:30.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.704962 2784 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-0/wal/instance
02:21:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.721755 2784 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu
02:21:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu
02:21:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-1/wal
02:21:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-1/data
02:21:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:21:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:21:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:21:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:21:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:21:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:21:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-1/logs
02:21:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-1/data/info.pb
02:21:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:21:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:21:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:21:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:21:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:21:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:21:30.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:21:30.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:21:30.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:21:30.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:21:30.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.184.2:0
02:21:30.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.184.2
02:21:30.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.184.2
02:21:30.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:21:30.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.184.62:44755,127.2.184.61:40497,127.2.184.60:37139
02:21:30.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.184.20:37367
02:21:30.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:21:30.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:21:30.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:21:30.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --scanner_ttl_ms=1000
02:21:30.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --scanner_gc_check_interval_us=500000 with env {}
02:21:30.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.761029 3219 heartbeater.cc:344] Connected to a master server at 127.2.184.60:37139
02:21:30.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.761600 3219 heartbeater.cc:461] Registering TS with master...
02:21:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.762912 3219 heartbeater.cc:507] Master 127.2.184.60:37139 requested a full tablet report, sending...
02:21:30.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.764909 3221 heartbeater.cc:344] Connected to a master server at 127.2.184.62:44755
02:21:30.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.765270 3221 heartbeater.cc:461] Registering TS with master...
02:21:30.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.765847 2985 ts_manager.cc:194] Registered new tserver with Master: 4bcdd8293835450d8de2c1e14364a517 (127.2.184.1:40481)
02:21:30.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.766244 3221 heartbeater.cc:507] Master 127.2.184.62:44755 requested a full tablet report, sending...
02:21:30.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.768450 2985 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.2.184.1:38651
02:21:30.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.769300 2830 ts_manager.cc:194] Registered new tserver with Master: 4bcdd8293835450d8de2c1e14364a517 (127.2.184.1:40481)
02:21:30.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.780659 3220 heartbeater.cc:344] Connected to a master server at 127.2.184.61:40497
02:21:30.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.781069 3220 heartbeater.cc:461] Registering TS with master...
02:21:30.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.781792 3220 heartbeater.cc:507] Master 127.2.184.61:40497 requested a full tablet report, sending...
02:21:30.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:30.784797 2905 ts_manager.cc:194] Registered new tserver with Master: 4bcdd8293835450d8de2c1e14364a517 (127.2.184.1:40481)
02:21:31.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:31.104877 3229 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:21:31.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:31.105563 3229 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:21:31.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:31.106217 3229 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:21:31.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:31.137009 3229 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:21:31.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:31.138015 3229 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.184.2
02:21:31.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:31.178006 3229 tablet_server_runner.cc:78] Tablet server non-default flags:
02:21:31.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:21:31.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.184.20:37367
02:21:31.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:21:31.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:21:31.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-1/data
02:21:31.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-1/wal
02:21:31.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:21:31.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:21:31.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.184.2:0
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-1/data/info.pb
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.184.2
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --scanner_gc_check_interval_us=500000
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --scanner_ttl_ms=1000
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.184.62:44755,127.2.184.61:40497,127.2.184.60:37139
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.184.2
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-1/logs
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7c956859e2f49c4cf1caffa969c1777a7a5d81c
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Jun 2025 01:43:20 UTC on 5fd53c4cbb9d
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6773
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:21:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:31.179643 3229 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:21:31.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:31.181528 3229 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:21:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:31.199203 3239 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:21:31.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:31.773420 3219 heartbeater.cc:499] Master 127.2.184.60:37139 was elected leader, sending a full tablet report...
02:21:32.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:31.199285 3236 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:21:32.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:31.199388 3237 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:21:32.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:32.377903 3238 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:21:32.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.377990 3229 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:21:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.382647 3229 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:21:32.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.385468 3229 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:21:32.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.386891 3229 hybrid_clock.cc:648] HybridClock initialized: now 1750904492386848 us; error 59 us; skew 500 ppm
02:21:32.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.387657 3229 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:21:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.393838 3229 webserver.cc:469] Webserver started at http://127.2.184.2:44777/ using document root <none> and password file <none>
02:21:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.394675 3229 fs_manager.cc:362] Metadata directory not provided
02:21:32.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.394852 3229 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:21:32.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.395345 3229 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
02:21:32.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.399664 3229 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-1/data/instance:
02:21:32.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "693c7b5c9ec9499ca6f5b6b11b5a313e"
02:21:32.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-26 02:21:32 on dist-test-slave-t7fr"
02:21:32.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.400779 3229 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-1/wal/instance:
02:21:32.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "693c7b5c9ec9499ca6f5b6b11b5a313e"
02:21:32.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-26 02:21:32 on dist-test-slave-t7fr"
02:21:32.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.407680 3229 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
02:21:32.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.413549 3247 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:21:32.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.414584 3229 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
02:21:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.414853 3229 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-1/data,/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-1/wal
02:21:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "693c7b5c9ec9499ca6f5b6b11b5a313e"
02:21:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-26 02:21:32 on dist-test-slave-t7fr"
02:21:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.415145 3229 fs_report.cc:389] FS layout report
02:21:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:21:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-1/wal
02:21:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-1/wal
02:21:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-1/data/data
02:21:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:21:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:21:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:21:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:21:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:21:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:21:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:21:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:21:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:21:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:21:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:21:32.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.469727 3229 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:21:32.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.471202 3229 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:21:32.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.471619 3229 kserver.cc:163] Server-wide thread pool size limit: 3276
02:21:32.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.474084 3229 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:21:32.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.477991 3229 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:21:32.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.478199 3229 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:21:32.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.478380 3229 ts_tablet_manager.cc:610] Registered 0 tablets
02:21:32.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.478508 3229 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:21:32.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.620503 3229 rpc_server.cc:307] RPC server started. Bound to: 127.2.184.2:46361
02:21:32.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.620618 3360 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.184.2:46361 every 8 connection(s)
02:21:32.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.623118 3229 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-1/data/info.pb
02:21:32.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.627225 2784 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu as pid 3229
02:21:32.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.627852 2784 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-1/wal/instance
02:21:32.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.657233 2784 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu
02:21:32.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu
02:21:32.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-2/wal
02:21:32.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-2/data
02:21:32.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:21:32.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:21:32.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:21:32.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:21:32.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-2/logs
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-2/data/info.pb
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.184.3:0
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.184.3
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.184.3
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.184.62:44755,127.2.184.61:40497,127.2.184.60:37139
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.184.20:37367
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --scanner_ttl_ms=1000
02:21:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --scanner_gc_check_interval_us=500000 with env {}
02:21:32.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.669167 3362 heartbeater.cc:344] Connected to a master server at 127.2.184.61:40497
02:21:32.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.669701 3362 heartbeater.cc:461] Registering TS with master...
02:21:32.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.670967 3362 heartbeater.cc:507] Master 127.2.184.61:40497 requested a full tablet report, sending...
02:21:32.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.674260 2905 ts_manager.cc:194] Registered new tserver with Master: 693c7b5c9ec9499ca6f5b6b11b5a313e (127.2.184.2:46361)
02:21:32.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.679643 3361 heartbeater.cc:344] Connected to a master server at 127.2.184.60:37139
02:21:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.679936 3361 heartbeater.cc:461] Registering TS with master...
02:21:32.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.680636 3363 heartbeater.cc:344] Connected to a master server at 127.2.184.62:44755
02:21:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.680784 3361 heartbeater.cc:507] Master 127.2.184.60:37139 requested a full tablet report, sending...
02:21:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.680970 3363 heartbeater.cc:461] Registering TS with master...
02:21:32.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.681864 3363 heartbeater.cc:507] Master 127.2.184.62:44755 requested a full tablet report, sending...
02:21:32.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.683182 2985 ts_manager.cc:194] Registered new tserver with Master: 693c7b5c9ec9499ca6f5b6b11b5a313e (127.2.184.2:46361)
02:21:32.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.684044 2830 ts_manager.cc:194] Registered new tserver with Master: 693c7b5c9ec9499ca6f5b6b11b5a313e (127.2.184.2:46361)
02:21:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:32.685055 2985 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.2.184.2:59835
02:21:32.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:32.981072 3371 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:21:32.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:32.981724 3371 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:21:32.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:32.982601 3371 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:21:33.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:33.030349 3371 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:21:33.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:33.031239 3371 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.184.3
02:21:33.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:33.066330 3371 tablet_server_runner.cc:78] Tablet server non-default flags:
02:21:33.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:21:33.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.184.20:37367
02:21:33.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:21:33.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:21:33.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-2/data
02:21:33.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-2/wal
02:21:33.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:21:33.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:21:33.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.184.3:0
02:21:33.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:21:33.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:21:33.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-2/data/info.pb
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.184.3
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --scanner_gc_check_interval_us=500000
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --scanner_ttl_ms=1000
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.184.62:44755,127.2.184.61:40497,127.2.184.60:37139
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.184.3
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-2/logs
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f7c956859e2f49c4cf1caffa969c1777a7a5d81c
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 26 Jun 2025 01:43:20 UTC on 5fd53c4cbb9d
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6773
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:21:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:33.067592 3371 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:21:33.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:33.069350 3371 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:21:33.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:33.087023 3379 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:21:33.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:33.688050 3361 heartbeater.cc:499] Master 127.2.184.60:37139 was elected leader, sending a full tablet report...
02:21:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==3371==WARNING: Can't read from symbolizer at fd 5
02:21:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==3371==WARNING: invalid path to external symbolizer!
02:21:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==3371==WARNING: Can't write to symbolizer at fd 10
02:21:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==3371==WARNING: Can't write to symbolizer at fd 10
02:21:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==3371==WARNING: Can't write to symbolizer at fd 10
02:21:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==3371==WARNING: Failed to use and restart external symbolizer!
02:21:34.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=3371)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 8 at 0x7b0400006d70 by thread T5:
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x49391f)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libc.so.6+0xd3fe4)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libglog.so.1+0x1f60b)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libglog.so.1+0x17b16)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libglog.so.1+0x18f10)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libkudu_cloud_util.so+0x11971)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libkudu_cloud_util.so+0x12c8d)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libkudu_cloud_util.so+0x12c19)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libkudu_cloud_util.so+0x12ba9)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (libkudu_cloud_util.so+0x12b71)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (libkudu_cloud_util.so+0x11e6d)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x59bcd4)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x59bb09)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_util.so+0x443636)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x465e5f)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous write of size 8 at 0x7b0400006d70 by thread T6:
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x49083c)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libc.so.6+0x9daf9)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libglog.so.1+0x1f60b)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libglog.so.1+0x17b16)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libglog.so.1+0x18f10)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libkudu_cloud_util.so+0x11971)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libkudu_cloud_util.so+0x12c8d)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libkudu_cloud_util.so+0x12c19)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libkudu_cloud_util.so+0x12ba9)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (libkudu_cloud_util.so+0x12b71)
02:21:34.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (libkudu_cloud_util.so+0x11e6d)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x59bcd4)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x59bb09)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_util.so+0x443636)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x465e5f)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T5 'AWS-3378' (tid=3378, running) created by main thread at:
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x469915)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libkudu_util.so+0x442e9a)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (kudu+0x58bf59)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libkudu_cloud_util.so+0x113f5)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libserver_process.so+0x145696)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libserver_process.so+0x143af6)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libkserver.so+0x12a01)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libtserver.so+0x255f92)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libtserver.so+0x263891)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x626289)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x527001)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x526f6c)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x526edc)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x525b45)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x13fc1f)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (libkudu_tools_util.so+0x132481)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (libkudu_tools_util.so+0x130212)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x641fc0)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (kudu+0x6428f7)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (kudu+0x642e3e)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (libc.so.6+0x21bf6)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T6 'Azure-3379' (tid=3379, finished) created by main thread at:
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x469915)
02:21:34.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libkudu_util.so+0x442e9a)
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (kudu+0x58bf59)
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libkudu_cloud_util.so+0x113f5)
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libserver_process.so+0x145696)
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libserver_process.so+0x143af6)
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libkserver.so+0x12a01)
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libtserver.so+0x255f92)
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libtserver.so+0x263891)
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x626289)
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x527001)
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x526f6c)
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x526edc)
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x525b45)
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x13fc1f)
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (libkudu_tools_util.so+0x132481)
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (libkudu_tools_util.so+0x130212)
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x641fc0)
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (kudu+0x6428f7)
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (kudu+0x642e3e)
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (libc.so.6+0x21bf6)
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu+0x49391f)
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:33.087071 3378 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:33.087404 3381 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:34.115159 3380 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
02:21:34.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=3371)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Read of size 8 at 0x7fc0406faa40 by thread T5:
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libkudu_util.so+0x443845)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libkudu_util.so+0x44dafc)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libkudu_util.so+0x443652)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (kudu+0x465e5f)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous write of size 8 at 0x7fc0406faa40 by thread T6 (mutexes: write M83):
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libglog.so.1+0x2b07e)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libkudu_util.so+0x443880)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libkudu_util.so+0x44dafc)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libkudu_util.so+0x443652)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (kudu+0x465e5f)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Location is global '<null>' at 0x000000000000 (libkudu_util.so+0x0000004eba40)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mutex M83 (0x7fc03e662840) created at:
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x49a0fb)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:34.115674 3371 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libglog.so.1+0xe6ec)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (ld-linux-x86-64.so.2+0x108f2)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T5 'AWS-3378' (tid=3378, running) created by main thread at:
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x469915)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libkudu_util.so+0x442e9a)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (kudu+0x58bf59)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libkudu_cloud_util.so+0x113f5)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libserver_process.so+0x145696)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libserver_process.so+0x143af6)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libkserver.so+0x12a01)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libtserver.so+0x255f92)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libtserver.so+0x263891)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x626289)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x527001)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x526f6c)
02:21:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x526edc)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x525b45)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x13fc1f)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (libkudu_tools_util.so+0x132481)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (libkudu_tools_util.so+0x130212)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x641fc0)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (kudu+0x6428f7)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (kudu+0x642e3e)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (libc.so.6+0x21bf6)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T6 'Azure-3379' (tid=3379, finished) created by main thread at:
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x469915)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libkudu_util.so+0x442e9a)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (kudu+0x58bf59)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libkudu_cloud_util.so+0x113f5)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libserver_process.so+0x145696)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libserver_process.so+0x143af6)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libkserver.so+0x12a01)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libtserver.so+0x255f92)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libtserver.so+0x263891)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x626289)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x527001)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x526f6c)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x526edc)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x525b45)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x13fc1f)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (libkudu_tools_util.so+0x132481)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (libkudu_tools_util.so+0x130212)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x641fc0)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (kudu+0x6428f7)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (kudu+0x642e3e)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (libc.so.6+0x21bf6)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-taskH7s0MS/build/tsan/lib/libkudu_util.so+0x443845)
02:21:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
02:21:34.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:34.118044 3371 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:21:34.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:34.120239 3371 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:21:34.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:34.121584 3371 hybrid_clock.cc:648] HybridClock initialized: now 1750904494121539 us; error 49 us; skew 500 ppm
02:21:34.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:34.122395 3371 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:21:34.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:34.128690 3371 webserver.cc:469] Webserver started at http://127.2.184.3:42249/ using document root <none> and password file <none>
02:21:34.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:34.129863 3371 fs_manager.cc:362] Metadata directory not provided
02:21:34.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:34.130070 3371 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:21:34.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:34.130302 3371 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
02:21:34.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:34.130816 3371 fs_manager.cc:362] Metadata directory not provided
02:21:34.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:34.130959 3371 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:21:34.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:34.131573 3371 ts_tablet_manager.cc:1500] Shutting down tablet manager...
02:21:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
02:21:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=3371)
02:21:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 4 at 0x7b6800001800 by main thread (mutexes: write M3747):
02:21:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1bfd34)
02:21:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x18e762)
02:21:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x1433f2)
02:21:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libserver_process.so+0x143186)
02:21:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmaster.so+0x4016df)
02:21:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libtserver.so+0x255922)
02:21:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libtserver.so+0x26391a)
02:21:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x626289)
02:21:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x527001)
02:21:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x526f6c)
02:21:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x526edc)
02:21:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x525b45)
02:21:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x13fc1f)
02:21:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x132481)
02:21:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x130212)
02:21:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x641fc0)
02:21:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x6428f7)
02:21:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x642e3e)
02:21:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libc.so.6+0x21bf6)
02:21:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous read of size 4 at 0x7b6800001800 by thread T12:
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1c171f)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (kudu+0x465e5f)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Location is heap block of size 1368 at 0x7b6800001800 allocated by main thread:
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x49607c)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c0244)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18f6c0)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libserver_process.so+0x143fee)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libkserver.so+0x12a01)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libtserver.so+0x255f92)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libtserver.so+0x263891)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x626289)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x527001)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x526f6c)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x526edc)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x525b45)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x13fc1f)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x132481)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x130212)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x641fc0)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x6428f7)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x642e3e)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libc.so.6+0x21bf6)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mutex M3747 (0x7b6800001978) created at:
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x469e82)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c0fc1)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18f6c0)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libserver_process.so+0x143fee)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libkserver.so+0x12a01)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libtserver.so+0x255f92)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libtserver.so+0x263891)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x626289)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x527001)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x526f6c)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x526edc)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x525b45)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x13fc1f)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x132481)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x130212)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x641fc0)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x6428f7)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x642e3e)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libc.so.6+0x21bf6)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T12 'sq_acceptor' (tid=3387, running) created by main thread at:
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x469915)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c14f7)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18f6c0)
02:21:34.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libserver_process.so+0x143fee)
02:21:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libkserver.so+0x12a01)
02:21:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libtserver.so+0x255f92)
02:21:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libtserver.so+0x263891)
02:21:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x626289)
02:21:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x527001)
02:21:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x526f6c)
02:21:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x526edc)
02:21:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x525b45)
02:21:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x13fc1f)
02:21:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x132481)
02:21:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x130212)
02:21:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x641fc0)
02:21:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x6428f7)
02:21:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x642e3e)
02:21:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libc.so.6+0x21bf6)
02:21:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-taskH7s0MS/build/tsan/lib/libserver_process.so+0x1bfd34)
02:21:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
02:21:34.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
02:21:34.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=3371)
02:21:34.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 8 at 0x7b0c00034140 by thread T12:
02:21:34.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x49391f)
02:21:34.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c17b0)
02:21:34.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (kudu+0x465e5f)
02:21:34.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:34.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous read of size 4 at 0x7b0c00034140 by main thread:
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1bab18)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x19027b)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18f7a8)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libserver_process.so+0x143fee)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libkserver.so+0x12a01)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libtserver.so+0x255f92)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libtserver.so+0x263891)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x626289)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x527001)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x526f6c)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x526edc)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x525b45)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x13fc1f)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x132481)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x130212)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x641fc0)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x6428f7)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x642e3e)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libc.so.6+0x21bf6)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T12 'sq_acceptor' (tid=3387, running) created by main thread at:
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x469915)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c14f7)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18f6c0)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libserver_process.so+0x143fee)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libkserver.so+0x12a01)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libtserver.so+0x255f92)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libtserver.so+0x263891)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x626289)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x527001)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x526f6c)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x526edc)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x525b45)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x13fc1f)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x132481)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x130212)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x641fc0)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x6428f7)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x642e3e)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libc.so.6+0x21bf6)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu+0x49391f)
02:21:34.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
02:21:34.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:34.145354 3371 builtin_ntp.cc:688] coult not shutdown socket: Network error: shutdown error: Transport endpoint is not connected (error 107)
02:21:34.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Could not create new FS layout: Write-ahead log directory /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-2/wal failed to canonicalize: Unable to canonicalize /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274/ts-2: No such file or directory (error 2)
02:21:35.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ThreadSanitizer: reported 4 warnings
02:21:35.186 [DEBUG - main] (MiniKuduCluster.java:195) Response: error {
code: RUNTIME_ERROR
message: "failed to start tablet server 3: /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu: process exited with non-zero status 66"
}
02:21:35.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:35.186508 2784 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu with pid 3070
02:21:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:35.209447 2784 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu with pid 3229
02:21:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:35.234951 2784 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu with pid 2796
02:21:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:35.261794 2784 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu with pid 2867
02:21:35.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:35.285866 2970 connection.cc:537] client connection to 127.2.184.61:40497 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
02:21:35.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250626 02:21:35.286378 2784 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu with pid 2943
02:21:35.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:35.310323 2784 env_posix.cc:2133] Unable to access file /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274 during walk: No such file or directory
02:21:35.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:35.310520 2784 env_posix.cc:2135] Unable to access file /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274 during walk: No such file or directory
02:21:35.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250626 02:21:35.310698 2784 tool_action_test.cc:443] Could not delete cluster root: IO error: /tmp/dist-test-taskH7s0MS/test-tmp/mini-kudu-cluster772266359746784274: One or more errors occurred
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==2784==WARNING: invalid path to external symbolizer!
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==2784==WARNING: Failed to use and restart external symbolizer!
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=2784)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 4 at 0x7b6800001e00 by main thread (mutexes: write M3450):
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1bfd34)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x18e762)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libmini_oidc.so+0x21aa2)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x219ea)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0xb3bbe)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libmini_cluster.so+0x9904d)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libmini_cluster.so+0x97feb)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libmini_cluster.so+0x8452a)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libmini_cluster.so+0x84689)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x653787)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x64fbed)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x64f2bb)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x64c4e0)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x527001)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x526f6c)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x526edc)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x525b45)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libkudu_tools_util.so+0x13fc1f)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libkudu_tools_util.so+0x132481)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (libkudu_tools_util.so+0x130212)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (kudu+0x641fc0)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #21 <null> <null> (kudu+0x6428f7)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #22 <null> <null> (kudu+0x642e3e)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #23 <null> <null> (libc.so.6+0x21bf6)
02:21:35.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous read of size 4 at 0x7b6800001e00 by thread T5:
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1c19a7)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (kudu+0x465e5f)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) As if synchronized via sleep:
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x48ae42)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libgutil.so+0xb5f0a)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libkudu_util.so+0x3d72a6)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_cluster.so+0x909fe)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x93ed7)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libmini_cluster.so+0x888cf)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libmini_cluster.so+0x86b0a)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x64dac9)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x64c3c4)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x527001)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x526f6c)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x526edc)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x525b45)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x13fc1f)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (libkudu_tools_util.so+0x132481)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (libkudu_tools_util.so+0x130212)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x641fc0)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x6428f7)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (kudu+0x642e3e)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (libc.so.6+0x21bf6)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Location is heap block of size 1368 at 0x7b6800001e00 allocated by main thread:
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x49607c)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c0244)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18f6c0)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x22183)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x85644)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x64dac9)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x64c3c4)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x527001)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x526f6c)
02:21:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x526edc)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x525b45)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13fc1f)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x132481)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x130212)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x641fc0)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6428f7)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x642e3e)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mutex M3450 (0x7b6800001f78) created at:
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x469e82)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c0fc1)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18f6c0)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x22183)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x85644)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x64dac9)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x64c3c4)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x527001)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x526f6c)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x526edc)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x525b45)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13fc1f)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x132481)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x130212)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x641fc0)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6428f7)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x642e3e)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T5 'sq_acceptor' (tid=2791, running) created by main thread at:
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x469915)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c14f7)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18f6c0)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x22183)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x85644)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x64dac9)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x64c3c4)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x527001)
02:21:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x526f6c)
02:21:35.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x526edc)
02:21:35.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x525b45)
02:21:35.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13fc1f)
02:21:35.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x132481)
02:21:35.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x130212)
02:21:35.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x641fc0)
02:21:35.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6428f7)
02:21:35.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x642e3e)
02:21:35.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
02:21:35.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:35.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-taskH7s0MS/build/tsan/lib/libserver_process.so+0x1bfd34)
02:21:35.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
02:21:35.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
02:21:35.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: ThreadSanitizer: data race (pid=2784)
02:21:35.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Write of size 4 at 0x7b6800001800 by main thread (mutexes: write M3447):
02:21:35.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1bfd34)
02:21:35.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x18e762)
02:21:35.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libmini_oidc.so+0x21ac5)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x219ea)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0xb3bbe)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libmini_cluster.so+0x9904d)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libmini_cluster.so+0x97feb)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libmini_cluster.so+0x8452a)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libmini_cluster.so+0x84689)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x653787)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x64fbed)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x64f2bb)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x64c4e0)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x527001)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x526f6c)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x526edc)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x525b45)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libkudu_tools_util.so+0x13fc1f)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libkudu_tools_util.so+0x132481)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (libkudu_tools_util.so+0x130212)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (kudu+0x641fc0)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #21 <null> <null> (kudu+0x6428f7)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #22 <null> <null> (kudu+0x642e3e)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #23 <null> <null> (libc.so.6+0x21bf6)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Previous read of size 4 at 0x7b6800001800 by thread T4:
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (libserver_process.so+0x1c19a7)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (kudu+0x465e5f)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) As if synchronized via sleep:
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x48a87d)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1bfdd9)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18e762)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x21aa2)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_oidc.so+0x219ea)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (libmini_cluster.so+0xb3bbe)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (libmini_cluster.so+0x9904d)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (libmini_cluster.so+0x97feb)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (libmini_cluster.so+0x8452a)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (libmini_cluster.so+0x84689)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x653787)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (kudu+0x64fbed)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (kudu+0x64f2bb)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (kudu+0x64c4e0)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x527001)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x526f6c)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x526edc)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (kudu+0x525b45)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #18 <null> <null> (libkudu_tools_util.so+0x13fc1f)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #19 <null> <null> (libkudu_tools_util.so+0x132481)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #20 <null> <null> (libkudu_tools_util.so+0x130212)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #21 <null> <null> (kudu+0x641fc0)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #22 <null> <null> (kudu+0x6428f7)
02:21:35.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #23 <null> <null> (kudu+0x642e3e)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #24 <null> <null> (libc.so.6+0x21bf6)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Location is heap block of size 1368 at 0x7b6800001800 allocated by main thread:
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x49607c)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c0244)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18f6c0)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x21dd0)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x85644)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x64dac9)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x64c3c4)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x527001)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x526f6c)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x526edc)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x525b45)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13fc1f)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x132481)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x130212)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x641fc0)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6428f7)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x642e3e)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mutex M3447 (0x7b6800001978) created at:
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x469e82)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c0fc1)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18f6c0)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x21dd0)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x85644)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x64dac9)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x64c3c4)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x527001)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x526f6c)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x526edc)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x525b45)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13fc1f)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x132481)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x130212)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x641fc0)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6428f7)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x642e3e)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Thread T4 'sq_acceptor' (tid=2790, running) created by main thread at:
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 <null> <null> (kudu+0x469915)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 <null> <null> (libserver_process.so+0x1c14f7)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 <null> <null> (libserver_process.so+0x18f6c0)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 <null> <null> (libmini_oidc.so+0x21dd0)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 <null> <null> (libmini_cluster.so+0x85644)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 <null> <null> (kudu+0x64dac9)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 <null> <null> (kudu+0x64c3c4)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 <null> <null> (kudu+0x527001)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 <null> <null> (kudu+0x526f6c)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 <null> <null> (kudu+0x526edc)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 <null> <null> (kudu+0x525b45)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 <null> <null> (libkudu_tools_util.so+0x13fc1f)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 <null> <null> (libkudu_tools_util.so+0x132481)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 <null> <null> (libkudu_tools_util.so+0x130212)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 <null> <null> (kudu+0x641fc0)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 <null> <null> (kudu+0x6428f7)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 <null> <null> (kudu+0x642e3e)
02:21:35.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 <null> <null> (libc.so.6+0x21bf6)
02:21:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:21:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: ThreadSanitizer: data race (/tmp/dist-test-taskH7s0MS/build/tsan/lib/libserver_process.so+0x1bfd34)
02:21:35.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ==================
02:21:35.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-26T02:21:35Z chronyd exiting
02:21:36.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ThreadSanitizer: reported 2 warnings
02:21:36.383 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduClient.testStopKeepAlivePeriodically: failed attempt 1
java.io.IOException: failed to start tablet server 3: /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu: process exited with non-zero status 66
at org.apache.kudu.test.cluster.MiniKuduCluster.sendRequestToCluster(MiniKuduCluster.java:199) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster.start(MiniKuduCluster.java:266) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster.access$300(MiniKuduCluster.java:85) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster$MiniKuduClusterBuilder.build(MiniKuduCluster.java:883) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:156) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.io.IOException: failed to start tablet server 3: /tmp/dist-test-taskH7s0MS/build/tsan/bin/kudu: process exited with non-zero status 66
at org.apache.kudu.test.cluster.MiniKuduCluster.sendRequestToCluster(MiniKuduCluster.java:199)
at org.apache.kudu.test.cluster.MiniKuduCluster.start(MiniKuduCluster.java:266)
at org.apache.kudu.test.cluster.MiniKuduCluster.access$300(MiniKuduCluster.java:85)
at org.apache.kudu.test.cluster.MiniKuduCluster$MiniKuduClusterBuilder.build(MiniKuduCluster.java:883)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:156)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)