03:20:43.242 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
03:20:43.243 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskgPZqXR/test-tmp
03:20:43.243 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:20:43.243 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskgPZqXR/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:20:43.244 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
03:20:43.680 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:20:43.680 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:20:43.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:43.684401 29322 mini_oidc.cc:150] Starting JWKS server
03:20:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:43.689930 29322 webserver.cc:458] Webserver started at http://127.0.0.1:42185/ using document root <none> and password file <none>
03:20:43.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:43.690498 29322 mini_oidc.cc:186] Starting OIDC Discovery server
03:20:43.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:43.693498 29322 webserver.cc:458] Webserver started at http://127.0.0.1:35255/ using document root <none> and password file <none>
03:20:43.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-13T03:20:43Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
03:20:43.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-13T03:20:43Z Disabled control of system clock
03:20:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:43.785288 29322 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu
03:20:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu
03:20:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-0/wal
03:20:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-0/data
03:20:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:20:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:20:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-0/logs
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-0/data/info.pb
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.162.190:42771
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.162.190
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.162.148:33613
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.162.190:42771,127.28.162.189:38647,127.28.162.188:42763
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:20:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42185/jwks.json with env {}
03:20:44.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:44.154999 29334 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:20:44.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:44.155663 29334 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:20:44.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:44.156252 29334 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:20:44.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:44.194160 29334 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:20:44.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:44.194489 29334 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:20:44.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:44.194697 29334 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:20:44.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:44.194900 29334 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:20:44.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:44.195097 29334 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:20:44.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:44.195242 29334 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42185/jwks.json
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:44.232946 29334 master_runner.cc:386] Master server non-default flags:
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.162.148:33613
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-0/data
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-0/wal
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.162.190:42771,127.28.162.189:38647,127.28.162.188:42763
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.162.190:42771
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42185/jwks.json
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-0/data/info.pb
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.162.190
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-0/logs
03:20:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:20:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:20:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:20:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:20:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:20:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
03:20:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:20:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 02:43:23 UTC on a633016ff5c9
03:20:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4152
03:20:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:20:44.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:44.234361 29334 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:20:44.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:44.235982 29334 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:20:44.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:44.254244 29340 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:20:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:44.254525 29341 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:20:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:44.257534 29334 server_base.cc:1034] running on GCE node
03:20:45.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:44.260812 29343 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:20:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.518707 29334 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:20:45.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.521677 29334 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:20:45.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.523108 29334 hybrid_clock.cc:648] HybridClock initialized: now 1736738445523061 us; error 64 us; skew 500 ppm
03:20:45.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.524037 29334 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:20:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.543262 29334 webserver.cc:458] Webserver started at http://127.28.162.190:43489/ using document root <none> and password file <none>
03:20:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.544284 29334 fs_manager.cc:362] Metadata directory not provided
03:20:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.544497 29334 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:20:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.544909 29334 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:20:45.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.550493 29334 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-0/data/instance:
03:20:45.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "693ae712a84e44ebb247810d5265e18a"
03:20:45.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 03:20:45 on dist-test-slave-d9vq"
03:20:45.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.552197 29334 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-0/wal/instance:
03:20:45.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "693ae712a84e44ebb247810d5265e18a"
03:20:45.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 03:20:45 on dist-test-slave-d9vq"
03:20:45.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.561873 29334 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.010s sys 0.001s
03:20:45.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.568946 29350 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:20:45.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.570447 29334 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.001s
03:20:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.570832 29334 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-0/data,/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-0/wal
03:20:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "693ae712a84e44ebb247810d5265e18a"
03:20:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 03:20:45 on dist-test-slave-d9vq"
03:20:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.571204 29334 fs_report.cc:389] FS layout report
03:20:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:20:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-0/wal
03:20:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-0/wal
03:20:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-0/data/data
03:20:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:20:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:20:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:20:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:20:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:20:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:20:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:20:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:20:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:20:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:20:45.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:20:45.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.627925 29334 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:20:45.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.629521 29334 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:20:45.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.629984 29334 kserver.cc:163] Server-wide thread pool size limit: 3276
03:20:45.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.719547 29334 rpc_server.cc:307] RPC server started. Bound to: 127.28.162.190:42771
03:20:45.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.719655 29402 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.162.190:42771 every 8 connection(s)
03:20:45.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.722815 29334 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-0/data/info.pb
03:20:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.728229 29322 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu as pid 29334
03:20:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.728727 29322 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-0/wal/instance
03:20:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.730242 29403 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:20:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.737087 29322 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu
03:20:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu
03:20:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-1/wal
03:20:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-1/data
03:20:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:20:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:20:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:20:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:20:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:20:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:20:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-1/logs
03:20:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-1/data/info.pb
03:20:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:20:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:20:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:20:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:20:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:20:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:20:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:20:45.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:20:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:20:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:20:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:20:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:20:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.162.189:38647
03:20:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.162.189
03:20:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:20:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.162.148:33613
03:20:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:20:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:20:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:20:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:20:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.162.190:42771,127.28.162.189:38647,127.28.162.188:42763
03:20:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:20:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42185/jwks.json with env {}
03:20:45.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.745270 29403 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.162.190" port: 42771 } has no permanent_uuid. Determining permanent_uuid...
03:20:45.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.775061 29403 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.162.189" port: 38647 } has no permanent_uuid. Determining permanent_uuid...
03:20:45.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:45.777597 29352 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.162.189:38647: connect: Connection refused (error 111)
03:20:45.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:45.782498 29403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.162.189:38647: Network error: Client connection negotiation failed: client connection to 127.28.162.189:38647: connect: Connection refused (error 111)
03:20:45.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.832201 29403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.162.189" port: 38647 } attempt: 1
03:20:45.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:45.837224 29403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.162.189:38647: Network error: Client connection negotiation failed: client connection to 127.28.162.189:38647: connect: Connection refused (error 111)
03:20:45.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:45.905817 29403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.162.189" port: 38647 } attempt: 2
03:20:45.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:45.910274 29403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.162.189:38647: Network error: Client connection negotiation failed: client connection to 127.28.162.189:38647: connect: Connection refused (error 111)
03:20:46.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:46.001807 29403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.162.189" port: 38647 } attempt: 3
03:20:46.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:46.005950 29403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.162.189:38647: Network error: Client connection negotiation failed: client connection to 127.28.162.189:38647: connect: Connection refused (error 111)
03:20:46.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:46.125905 29405 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:20:46.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:46.126497 29405 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:20:46.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:46.126993 29405 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:20:46.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:46.149446 29403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.162.189" port: 38647 } attempt: 4
03:20:46.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:46.155100 29403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.162.189:38647: Network error: Client connection negotiation failed: client connection to 127.28.162.189:38647: connect: Connection refused (error 111)
03:20:46.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:46.168675 29405 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:20:46.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:46.169018 29405 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:20:46.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:46.169298 29405 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:20:46.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:46.169515 29405 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:20:46.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:46.169819 29405 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:20:46.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:46.170083 29405 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42185/jwks.json
03:20:46.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:46.210225 29405 master_runner.cc:386] Master server non-default flags:
03:20:46.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:20:46.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.162.148:33613
03:20:46.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:20:46.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:20:46.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-1/data
03:20:46.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-1/wal
03:20:46.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:20:46.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.162.190:42771,127.28.162.189:38647,127.28.162.188:42763
03:20:46.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:20:46.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:20:46.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.162.189:38647
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42185/jwks.json
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-1/data/info.pb
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.162.189
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-1/logs
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 02:43:23 UTC on a633016ff5c9
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4152
03:20:46.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:20:46.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:46.211941 29405 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:20:46.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:46.213835 29405 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:20:46.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:46.231266 29417 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:20:46.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:46.454706 29403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.162.189" port: 38647 } attempt: 5
03:20:46.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:46.459700 29403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.162.189:38647: Network error: Client connection negotiation failed: client connection to 127.28.162.189:38647: connect: Connection refused (error 111)
03:20:47.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.007150 29403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.162.189" port: 38647 } attempt: 6
03:20:47.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:47.014796 29403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.162.189:38647: Network error: Client connection negotiation failed: client connection to 127.28.162.189:38647: connect: Connection refused (error 111)
03:20:47.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:46.231632 29414 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:20:47.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:46.232517 29415 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:20:47.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:46.233647 29405 server_base.cc:1034] running on GCE node
03:20:47.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.494175 29405 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:20:47.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.498682 29405 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:20:47.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.500185 29405 hybrid_clock.cc:648] HybridClock initialized: now 1736738447500138 us; error 53 us; skew 500 ppm
03:20:47.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.501036 29405 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:20:47.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.508546 29405 webserver.cc:458] Webserver started at http://127.28.162.189:37155/ using document root <none> and password file <none>
03:20:47.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.509632 29405 fs_manager.cc:362] Metadata directory not provided
03:20:47.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.509886 29405 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:20:47.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.510301 29405 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:20:47.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.516104 29405 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-1/data/instance:
03:20:47.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3fd31af5b2b944caa36533b531181e55"
03:20:47.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 03:20:47 on dist-test-slave-d9vq"
03:20:47.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.517354 29405 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-1/wal/instance:
03:20:47.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3fd31af5b2b944caa36533b531181e55"
03:20:47.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 03:20:47 on dist-test-slave-d9vq"
03:20:47.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.525235 29405 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
03:20:47.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.531931 29426 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:20:47.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.533365 29405 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.001s
03:20:47.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.533705 29405 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-1/data,/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-1/wal
03:20:47.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3fd31af5b2b944caa36533b531181e55"
03:20:47.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 03:20:47 on dist-test-slave-d9vq"
03:20:47.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.534078 29405 fs_report.cc:389] FS layout report
03:20:47.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:20:47.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-1/wal
03:20:47.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-1/wal
03:20:47.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-1/data/data
03:20:47.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:20:47.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:20:47.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:20:47.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:20:47.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:20:47.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:20:47.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:20:47.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:20:47.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:20:47.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:20:47.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:20:47.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.618626 29405 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:20:47.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.620208 29405 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:20:47.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.620628 29405 kserver.cc:163] Server-wide thread pool size limit: 3276
03:20:47.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.694896 29405 rpc_server.cc:307] RPC server started. Bound to: 127.28.162.189:38647
03:20:47.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.694983 29477 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.162.189:38647 every 8 connection(s)
03:20:47.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.697824 29405 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-1/data/info.pb
03:20:47.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.702328 29322 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu as pid 29405
03:20:47.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.702834 29322 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-1/wal/instance
03:20:47.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.704118 29478 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.705987 29322 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-2/wal
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-2/data
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-2/logs
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-2/data/info.pb
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.162.188:42763
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.162.188
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:20:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.162.148:33613
03:20:47.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:20:47.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:20:47.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:20:47.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:20:47.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.162.190:42771,127.28.162.189:38647,127.28.162.188:42763
03:20:47.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:20:47.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42185/jwks.json with env {}
03:20:47.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.721568 29478 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.162.190" port: 42771 } has no permanent_uuid. Determining permanent_uuid...
03:20:47.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.747284 29478 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.162.189" port: 38647 } has no permanent_uuid. Determining permanent_uuid...
03:20:47.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.763288 29478 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } has no permanent_uuid. Determining permanent_uuid...
03:20:47.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:47.766203 29429 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.162.188:42763: connect: Connection refused (error 111)
03:20:47.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:47.770535 29478 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.162.188:42763: Network error: Client connection negotiation failed: client connection to 127.28.162.188:42763: connect: Connection refused (error 111)
03:20:47.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.820256 29478 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } attempt: 1
03:20:47.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:47.824652 29478 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.162.188:42763: Network error: Client connection negotiation failed: client connection to 127.28.162.188:42763: connect: Connection refused (error 111)
03:20:47.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.893123 29478 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } attempt: 2
03:20:47.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:47.897078 29478 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.162.188:42763: Network error: Client connection negotiation failed: client connection to 127.28.162.188:42763: connect: Connection refused (error 111)
03:20:47.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:47.988729 29478 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } attempt: 3
03:20:47.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:47.993350 29478 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.162.188:42763: Network error: Client connection negotiation failed: client connection to 127.28.162.188:42763: connect: Connection refused (error 111)
03:20:48.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:48.066457 29480 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:20:48.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:48.067071 29480 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:20:48.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:48.067917 29480 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:20:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:48.075235 29403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.162.189" port: 38647 } attempt: 7
03:20:48.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:48.088294 29403 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } has no permanent_uuid. Determining permanent_uuid...
03:20:48.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:48.094168 29403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.162.188:42763: Network error: Client connection negotiation failed: client connection to 127.28.162.188:42763: connect: Connection refused (error 111)
03:20:48.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:48.106633 29480 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:20:48.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:48.107038 29480 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:20:48.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:48.107260 29480 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:20:48.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:48.107476 29480 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:20:48.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:48.107692 29480 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:20:48.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:48.108103 29480 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42185/jwks.json
03:20:48.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:48.136906 29478 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } attempt: 4
03:20:48.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:48.142046 29478 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.162.188:42763: Network error: Client connection negotiation failed: client connection to 127.28.162.188:42763: connect: Connection refused (error 111)
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:48.149681 29480 master_runner.cc:386] Master server non-default flags:
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.162.148:33613
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-2/data
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-2/wal
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.162.190:42771,127.28.162.189:38647,127.28.162.188:42763
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.162.188:42763
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42185/jwks.json
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-2/data/info.pb
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.162.188
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:20:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:20:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:20:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-2/logs
03:20:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:20:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:20:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:20:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:20:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:20:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
03:20:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:20:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 02:43:23 UTC on a633016ff5c9
03:20:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4152
03:20:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:20:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:48.151055 29480 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:20:48.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:48.152750 29480 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:20:48.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:48.152715 29403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } attempt: 1
03:20:48.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:48.157240 29403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.162.188:42763: Network error: Client connection negotiation failed: client connection to 127.28.162.188:42763: connect: Connection refused (error 111)
03:20:48.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:48.173964 29493 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:20:48.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:48.238780 29403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } attempt: 2
03:20:48.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:48.244602 29403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.162.188:42763: Network error: Client connection negotiation failed: client connection to 127.28.162.188:42763: connect: Connection refused (error 111)
03:20:48.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:48.330093 29403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } attempt: 3
03:20:48.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:48.335896 29403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.162.188:42763: Network error: Client connection negotiation failed: client connection to 127.28.162.188:42763: connect: Connection refused (error 111)
03:20:48.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:48.441558 29478 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } attempt: 5
03:20:48.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:48.447011 29478 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.162.188:42763: Network error: Client connection negotiation failed: client connection to 127.28.162.188:42763: connect: Connection refused (error 111)
03:20:48.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:48.476385 29403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } attempt: 4
03:20:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:48.482311 29403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.162.188:42763: Network error: Client connection negotiation failed: client connection to 127.28.162.188:42763: connect: Connection refused (error 111)
03:20:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:48.765808 29403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } attempt: 5
03:20:48.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:48.772434 29403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.162.188:42763: Network error: Client connection negotiation failed: client connection to 127.28.162.188:42763: connect: Connection refused (error 111)
03:20:48.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:48.994679 29478 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } attempt: 6
03:20:49.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:49.003257 29478 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.162.188:42763: Network error: Client connection negotiation failed: client connection to 127.28.162.188:42763: connect: Connection refused (error 111)
03:20:49.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.324867 29403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } attempt: 6
03:20:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:49.335338 29403 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.162.188:42763: Network error: Client connection negotiation failed: client connection to 127.28.162.188:42763: connect: Connection refused (error 111)
03:20:49.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:48.175717 29492 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:20:49.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:48.176229 29496 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:20:49.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:48.177158 29480 server_base.cc:1034] running on GCE node
03:20:49.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.484967 29480 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:20:49.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.488065 29480 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:20:49.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.490495 29480 hybrid_clock.cc:648] HybridClock initialized: now 1736738449490408 us; error 95 us; skew 500 ppm
03:20:49.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.492436 29480 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:20:49.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.501412 29480 webserver.cc:458] Webserver started at http://127.28.162.188:45733/ using document root <none> and password file <none>
03:20:49.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.502640 29480 fs_manager.cc:362] Metadata directory not provided
03:20:49.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.502934 29480 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:20:49.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.504148 29480 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:20:49.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.511969 29480 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-2/data/instance:
03:20:49.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7c16e7dc4cee4c6fbd69462a9ae25f8d"
03:20:49.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 03:20:49 on dist-test-slave-d9vq"
03:20:49.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.513571 29480 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-2/wal/instance:
03:20:49.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7c16e7dc4cee4c6fbd69462a9ae25f8d"
03:20:49.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 03:20:49 on dist-test-slave-d9vq"
03:20:49.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.521988 29480 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
03:20:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.528990 29507 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:20:49.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.530441 29480 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
03:20:49.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.530783 29480 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-2/data,/tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-2/wal
03:20:49.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7c16e7dc4cee4c6fbd69462a9ae25f8d"
03:20:49.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 03:20:49 on dist-test-slave-d9vq"
03:20:49.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.531123 29480 fs_report.cc:389] FS layout report
03:20:49.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:20:49.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-2/wal
03:20:49.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-2/wal
03:20:49.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-2/data/data
03:20:49.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:20:49.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:20:49.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:20:49.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:20:49.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:20:49.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:20:49.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:20:49.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:20:49.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:20:49.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:20:49.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:20:49.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.587628 29480 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:20:49.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.589447 29480 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:20:49.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.589926 29480 kserver.cc:163] Server-wide thread pool size limit: 3276
03:20:49.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.668862 29480 rpc_server.cc:307] RPC server started. Bound to: 127.28.162.188:42763
03:20:49.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.668988 29558 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.162.188:42763 every 8 connection(s)
03:20:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.672425 29480 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-2/data/info.pb
03:20:49.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.679991 29322 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu as pid 29480
03:20:49.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.680024 29559 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:20:49.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.681043 29322 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-2/wal/instance
03:20:49.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.704836 29559 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.162.190" port: 42771 } has no permanent_uuid. Determining permanent_uuid...
03:20:49.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.747947 29559 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.162.189" port: 38647 } has no permanent_uuid. Determining permanent_uuid...
03:20:49.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.762077 29559 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } has no permanent_uuid. Determining permanent_uuid...
03:20:49.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.790047 29559 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7c16e7dc4cee4c6fbd69462a9ae25f8d: Bootstrap starting.
03:20:49.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.800873 29559 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7c16e7dc4cee4c6fbd69462a9ae25f8d: Neither blocks nor log segments found. Creating new log.
03:20:49.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.802917 29559 log.cc:826] T 00000000000000000000000000000000 P 7c16e7dc4cee4c6fbd69462a9ae25f8d: Log is configured to *not* fsync() on all Append() calls
03:20:49.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.808785 29559 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7c16e7dc4cee4c6fbd69462a9ae25f8d: No bootstrap required, opened a new log
03:20:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.840690 29559 raft_consensus.cc:357] T 00000000000000000000000000000000 P 7c16e7dc4cee4c6fbd69462a9ae25f8d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "693ae712a84e44ebb247810d5265e18a" member_type: VOTER last_known_addr { host: "127.28.162.190" port: 42771 } } peers { permanent_uuid: "3fd31af5b2b944caa36533b531181e55" member_type: VOTER last_known_addr { host: "127.28.162.189" port: 38647 } } peers { permanent_uuid: "7c16e7dc4cee4c6fbd69462a9ae25f8d" member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } }
03:20:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.841478 29559 raft_consensus.cc:383] T 00000000000000000000000000000000 P 7c16e7dc4cee4c6fbd69462a9ae25f8d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:20:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.841774 29559 raft_consensus.cc:738] T 00000000000000000000000000000000 P 7c16e7dc4cee4c6fbd69462a9ae25f8d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7c16e7dc4cee4c6fbd69462a9ae25f8d, State: Initialized, Role: FOLLOWER
03:20:49.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.842625 29559 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7c16e7dc4cee4c6fbd69462a9ae25f8d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "693ae712a84e44ebb247810d5265e18a" member_type: VOTER last_known_addr { host: "127.28.162.190" port: 42771 } } peers { permanent_uuid: "3fd31af5b2b944caa36533b531181e55" member_type: VOTER last_known_addr { host: "127.28.162.189" port: 38647 } } peers { permanent_uuid: "7c16e7dc4cee4c6fbd69462a9ae25f8d" member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } }
03:20:49.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.847239 29559 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7c16e7dc4cee4c6fbd69462a9ae25f8d [sys.catalog]: configured and running, proceeding with master startup.
03:20:49.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.845458 29570 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7c16e7dc4cee4c6fbd69462a9ae25f8d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "693ae712a84e44ebb247810d5265e18a" member_type: VOTER last_known_addr { host: "127.28.162.190" port: 42771 } } peers { permanent_uuid: "3fd31af5b2b944caa36533b531181e55" member_type: VOTER last_known_addr { host: "127.28.162.189" port: 38647 } } peers { permanent_uuid: "7c16e7dc4cee4c6fbd69462a9ae25f8d" member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } } }
03:20:49.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.848018 29570 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7c16e7dc4cee4c6fbd69462a9ae25f8d [sys.catalog]: This master's current role is: FOLLOWER
03:20:49.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:49.890173 29581 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 7c16e7dc4cee4c6fbd69462a9ae25f8d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:20:49.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:49.890611 29581 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:20:49.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.881871 29480 master_runner.cc:186] Error getting master registration for 127.28.162.190:42771: OK, instance_id { permanent_uuid: "693ae712a84e44ebb247810d5265e18a" instance_seqno: 1736738445693054 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:20:49.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.921329 29480 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:20:49.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.980176 29570 raft_consensus.cc:491] T 00000000000000000000000000000000 P 7c16e7dc4cee4c6fbd69462a9ae25f8d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:20:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.980795 29570 raft_consensus.cc:513] T 00000000000000000000000000000000 P 7c16e7dc4cee4c6fbd69462a9ae25f8d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "693ae712a84e44ebb247810d5265e18a" member_type: VOTER last_known_addr { host: "127.28.162.190" port: 42771 } } peers { permanent_uuid: "3fd31af5b2b944caa36533b531181e55" member_type: VOTER last_known_addr { host: "127.28.162.189" port: 38647 } } peers { permanent_uuid: "7c16e7dc4cee4c6fbd69462a9ae25f8d" member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } }
03:20:49.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:49.985656 29570 leader_election.cc:290] T 00000000000000000000000000000000 P 7c16e7dc4cee4c6fbd69462a9ae25f8d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 693ae712a84e44ebb247810d5265e18a (127.28.162.190:42771), 3fd31af5b2b944caa36533b531181e55 (127.28.162.189:38647)
03:20:50.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.064474 29478 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } attempt: 7
03:20:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:50.096992 29513 tablet.cc:2367] T 00000000000000000000000000000000 P 7c16e7dc4cee4c6fbd69462a9ae25f8d: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:20:50.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.108775 29478 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55: Bootstrap starting.
03:20:50.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.118525 29478 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55: Neither blocks nor log segments found. Creating new log.
03:20:50.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.122005 29478 log.cc:826] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55: Log is configured to *not* fsync() on all Append() calls
03:20:50.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.134326 29478 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55: No bootstrap required, opened a new log
03:20:50.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.182147 29478 raft_consensus.cc:357] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "693ae712a84e44ebb247810d5265e18a" member_type: VOTER last_known_addr { host: "127.28.162.190" port: 42771 } } peers { permanent_uuid: "3fd31af5b2b944caa36533b531181e55" member_type: VOTER last_known_addr { host: "127.28.162.189" port: 38647 } } peers { permanent_uuid: "7c16e7dc4cee4c6fbd69462a9ae25f8d" member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } }
03:20:50.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.183585 29478 raft_consensus.cc:383] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:20:50.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.184505 29478 raft_consensus.cc:738] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3fd31af5b2b944caa36533b531181e55, State: Initialized, Role: FOLLOWER
03:20:50.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.186520 29478 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "693ae712a84e44ebb247810d5265e18a" member_type: VOTER last_known_addr { host: "127.28.162.190" port: 42771 } } peers { permanent_uuid: "3fd31af5b2b944caa36533b531181e55" member_type: VOTER last_known_addr { host: "127.28.162.189" port: 38647 } } peers { permanent_uuid: "7c16e7dc4cee4c6fbd69462a9ae25f8d" member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } }
03:20:50.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.194293 29478 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55 [sys.catalog]: configured and running, proceeding with master startup.
03:20:50.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.194548 29453 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7c16e7dc4cee4c6fbd69462a9ae25f8d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3fd31af5b2b944caa36533b531181e55" is_pre_election: true
03:20:50.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.195578 29453 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7c16e7dc4cee4c6fbd69462a9ae25f8d in term 0.
03:20:50.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.197721 29509 leader_election.cc:304] T 00000000000000000000000000000000 P 7c16e7dc4cee4c6fbd69462a9ae25f8d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3fd31af5b2b944caa36533b531181e55, 7c16e7dc4cee4c6fbd69462a9ae25f8d; no voters:
03:20:50.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.198828 29570 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 7c16e7dc4cee4c6fbd69462a9ae25f8d [term 0 FOLLOWER]: Leader pre-election won for term 1
03:20:50.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.199175 29570 raft_consensus.cc:491] T 00000000000000000000000000000000 P 7c16e7dc4cee4c6fbd69462a9ae25f8d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:20:50.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.199483 29570 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 7c16e7dc4cee4c6fbd69462a9ae25f8d [term 0 FOLLOWER]: Advancing to term 1
03:20:50.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) F20250113 03:20:50.200345 29570 raft_consensus.cc:3219] Check failed: _s.ok() Bad status: Not found: Unable to create consensus metadata root dir: /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-2/wal/consensus-meta: No such file or directory (error 2)
03:20:50.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.192782 29585 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "693ae712a84e44ebb247810d5265e18a" member_type: VOTER last_known_addr { host: "127.28.162.190" port: 42771 } } peers { permanent_uuid: "3fd31af5b2b944caa36533b531181e55" member_type: VOTER last_known_addr { host: "127.28.162.189" port: 38647 } } peers { permanent_uuid: "7c16e7dc4cee4c6fbd69462a9ae25f8d" member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } } }
03:20:50.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.222992 29585 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55 [sys.catalog]: This master's current role is: FOLLOWER
03:20:50.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) *** Check failure stack trace: ***
03:20:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5e22596950 (/tmp/dist-test-taskgPZqXR/thirdparty/installed/tsan/lib/libglog.so.0.6.0 (deleted)+0x1994f)
03:20:50.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5e2259ba8b (/tmp/dist-test-taskgPZqXR/thirdparty/installed/tsan/lib/libglog.so.0.6.0 (deleted)+0x1ea8a)
03:20:50.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:50.286368 29596 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:20:50.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:50.287142 29596 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:20:50.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5e2b73af72 (/tmp/dist-test-taskgPZqXR/build/tsan/lib/libconsensus.so (deleted)+0x196f71)
03:20:50.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5e2b7389da (/tmp/dist-test-taskgPZqXR/build/tsan/lib/libconsensus.so (deleted)+0x1949d9)
03:20:50.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.268832 29405 master_runner.cc:186] Error getting master registration for 127.28.162.190:42771: OK, instance_id { permanent_uuid: "693ae712a84e44ebb247810d5265e18a" instance_seqno: 1736738445693054 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:20:50.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.349526 29405 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:20:50.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5e2b74ad13 (/tmp/dist-test-taskgPZqXR/build/tsan/lib/libconsensus.so (deleted)+0x1a6d12)
03:20:50.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.368881 29403 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } attempt: 7
03:20:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5e2b75ed66 (/tmp/dist-test-taskgPZqXR/build/tsan/lib/libconsensus.so (deleted)+0x1bad65)
03:20:50.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:50.382416 29432 tablet.cc:2367] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:20:50.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20250113 03:20:50.401057 29403 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to persist consensus metadata for tablet 00000000000000000000000000000000: Unable to create consensus metadata for tablet 00000000000000000000000000000000: Unable to create consensus metadata root dir: /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-0/wal/consensus-meta: No such file or directory (error 2)
03:20:50.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.401619 29334 master.cc:537] Master@127.28.162.190:42771 shutting down...
03:20:50.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:50.404227 29509 proxy.cc:239] Call had error, refreshing address and retrying: Remote error: Service unavailable: Systable not yet initialized
03:20:50.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5e2b75ed0a (/tmp/dist-test-taskgPZqXR/build/tsan/lib/libconsensus.so (deleted)+0x1bad09)
03:20:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5e2b75ec9a (/tmp/dist-test-taskgPZqXR/build/tsan/lib/libconsensus.so (deleted)+0x1bac99)
03:20:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5e2b75ec62 (/tmp/dist-test-taskgPZqXR/build/tsan/lib/libconsensus.so (deleted)+0x1bac61)
03:20:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:50.471391 29509 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.28.162.190:42771, user_credentials={real_user=slave}} blocked reactor thread for 67192.5us
03:20:50.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:50.473181 29509 leader_election.cc:336] T 00000000000000000000000000000000 P 7c16e7dc4cee4c6fbd69462a9ae25f8d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 693ae712a84e44ebb247810d5265e18a (127.28.162.190:42771): Network error: recv got EOF from unknown peer (error 108)
03:20:50.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5e2b75defe (/tmp/dist-test-taskgPZqXR/build/tsan/lib/libconsensus.so (deleted)+0x1b9efd)
03:20:50.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x59ea05 (/tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu (deleted)+0x59ea04)
03:20:50.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x59e83a (/tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu (deleted)+0x59e839)
03:20:50.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5e247f9157 (/tmp/dist-test-taskgPZqXR/build/tsan/lib/libkudu_util.so (deleted)+0x468156)
03:20:50.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5e247fc5b2 (/tmp/dist-test-taskgPZqXR/build/tsan/lib/libkudu_util.so (deleted)+0x46b5b1)
03:20:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5e247fc56a (/tmp/dist-test-taskgPZqXR/build/tsan/lib/libkudu_util.so (deleted)+0x46b569)
03:20:50.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:50.542554 29334 rolling_log.cc:182] Unable to compress old log file: Not found: Unable to open input file to compress: /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-0/logs/kudu-master.dist-test-slave-d9vq.slave.diagnostics.20250113-032045.0.29334: No such file or directory (error 2)
03:20:50.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:50.543336 29334 rolling_log.cc:186] failed to delete old diagnostics log files: IO error: glob failed for /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-0/logs/kudu-master.dist-test-slave-d9vq.slave.diagnostics.*.*.*: No such file or directory
03:20:50.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.543819 29334 master.cc:559] Master@127.28.162.190:42771 shutdown complete.
03:20:50.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5e247fc4fa (/tmp/dist-test-taskgPZqXR/build/tsan/lib/libkudu_util.so (deleted)+0x46b4f9)
03:20:50.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to persist consensus metadata for tablet 00000000000000000000000000000000: Unable to create consensus metadata for tablet 00000000000000000000000000000000: Unable to create consensus metadata root dir: /tmp/dist-test-taskgPZqXR/test-tmp/mini-kudu-cluster6421807558514806032/master-0/wal/consensus-meta: No such file or directory (error 2)
03:20:50.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:50.558032 29334 logging.cc:424] LogThrottler /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/proxy.cc:239: suppressed but not reported on 13 messages since previous log ~4 seconds ago
03:20:50.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5e247fc4c2 (/tmp/dist-test-taskgPZqXR/build/tsan/lib/libkudu_util.so (deleted)+0x46b4c1)
03:20:50.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5e247fb7be (/tmp/dist-test-taskgPZqXR/build/tsan/lib/libkudu_util.so (deleted)+0x46a7bd)
03:20:50.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x59ea05 (/tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu (deleted)+0x59ea04)
03:20:50.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x59e83a (/tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu (deleted)+0x59e839)
03:20:50.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5e247db8d7 (/tmp/dist-test-taskgPZqXR/build/tsan/lib/libkudu_util.so (deleted)+0x44a8d6)
03:20:50.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x468a00 (/tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu (deleted)+0x4689ff)
03:20:50.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5e25e5a6db start_thread
03:20:50.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f5e2123b71f clone
03:20:51.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:51.291859 29596 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:20:51.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:51.292255 29596 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:20:52.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:52.260383 29601 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:20:52.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:52.260809 29601 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "693ae712a84e44ebb247810d5265e18a" member_type: VOTER last_known_addr { host: "127.28.162.190" port: 42771 } } peers { permanent_uuid: "3fd31af5b2b944caa36533b531181e55" member_type: VOTER last_known_addr { host: "127.28.162.189" port: 38647 } } peers { permanent_uuid: "7c16e7dc4cee4c6fbd69462a9ae25f8d" member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } }
03:20:52.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:52.262974 29601 leader_election.cc:290] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 693ae712a84e44ebb247810d5265e18a (127.28.162.190:42771), 7c16e7dc4cee4c6fbd69462a9ae25f8d (127.28.162.188:42763)
03:20:52.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:52.270722 29428 leader_election.cc:336] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 693ae712a84e44ebb247810d5265e18a (127.28.162.190:42771): Network error: Client connection negotiation failed: client connection to 127.28.162.190:42771: connect: Connection refused (error 111)
03:20:52.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:52.271498 29429 leader_election.cc:336] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7c16e7dc4cee4c6fbd69462a9ae25f8d (127.28.162.188:42763): Network error: Client connection negotiation failed: client connection to 127.28.162.188:42763: connect: Connection refused (error 111)
03:20:52.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:52.272094 29429 leader_election.cc:304] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3fd31af5b2b944caa36533b531181e55; no voters: 693ae712a84e44ebb247810d5265e18a, 7c16e7dc4cee4c6fbd69462a9ae25f8d
03:20:52.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:52.273061 29601 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:20:52.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:52.296692 29596 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:20:52.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:52.297096 29596 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:20:53.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:53.300807 29596 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:20:53.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:53.301121 29596 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:20:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:54.163022 29606 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:20:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:54.163468 29606 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "693ae712a84e44ebb247810d5265e18a" member_type: VOTER last_known_addr { host: "127.28.162.190" port: 42771 } } peers { permanent_uuid: "3fd31af5b2b944caa36533b531181e55" member_type: VOTER last_known_addr { host: "127.28.162.189" port: 38647 } } peers { permanent_uuid: "7c16e7dc4cee4c6fbd69462a9ae25f8d" member_type: VOTER last_known_addr { host: "127.28.162.188" port: 42763 } }
03:20:54.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:54.165174 29606 leader_election.cc:290] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 693ae712a84e44ebb247810d5265e18a (127.28.162.190:42771), 7c16e7dc4cee4c6fbd69462a9ae25f8d (127.28.162.188:42763)
03:20:54.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:54.167886 29429 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.162.188:42763: connect: Connection refused (error 111) [suppressed 8 similar messages]
03:20:54.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:54.173266 29429 leader_election.cc:336] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7c16e7dc4cee4c6fbd69462a9ae25f8d (127.28.162.188:42763): Network error: Client connection negotiation failed: client connection to 127.28.162.188:42763: connect: Connection refused (error 111)
03:20:54.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:54.173864 29428 leader_election.cc:336] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 693ae712a84e44ebb247810d5265e18a (127.28.162.190:42771): Network error: Client connection negotiation failed: client connection to 127.28.162.190:42771: connect: Connection refused (error 111)
03:20:54.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:54.174329 29428 leader_election.cc:304] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3fd31af5b2b944caa36533b531181e55; no voters: 693ae712a84e44ebb247810d5265e18a, 7c16e7dc4cee4c6fbd69462a9ae25f8d
03:20:54.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 03:20:54.175000 29606 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:20:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:54.304176 29596 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 3fd31af5b2b944caa36533b531181e55: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:20:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 03:20:54.304500 29596 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:20:54.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) F20250113 03:20:54.704129 29322 external_mini_cluster.cc:638] Check failed: leader_master() != nullptr Must have started at least 1 master before adding tablet servers
03:20:54.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) *** Check failure stack trace: ***
03:20:54.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fa6a93da950 (/tmp/dist-test-taskgPZqXR/thirdparty/installed/tsan/lib/libglog.so.0.6.0 (deleted)+0x1994f)
03:20:54.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fa6a93dfa8b (/tmp/dist-test-taskgPZqXR/thirdparty/installed/tsan/lib/libglog.so.0.6.0 (deleted)+0x1ea8a)
03:20:54.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fa6b47bcc1d (/tmp/dist-test-taskgPZqXR/build/tsan/lib/libmini_cluster.so (deleted)+0x88c1c)
03:20:54.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fa6b47bac9b (/tmp/dist-test-taskgPZqXR/build/tsan/lib/libmini_cluster.so (deleted)+0x86c9a)
03:20:54.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x6510aa (/tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu (deleted)+0x6510a9)
03:20:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x64f9a5 (/tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu (deleted)+0x64f9a4)
03:20:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x529a02 (/tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu (deleted)+0x529a01)
03:20:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x52996d (/tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu (deleted)+0x52996c)
03:20:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x5298dd (/tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu (deleted)+0x5298dc)
03:20:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x528546 (/tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu (deleted)+0x528545)
03:20:54.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fa6b452df90 (/tmp/dist-test-taskgPZqXR/build/tsan/lib/libkudu_tools_util.so (deleted)+0x13df8f)
03:20:54.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fa6b45207f2 (/tmp/dist-test-taskgPZqXR/build/tsan/lib/libkudu_tools_util.so (deleted)+0x1307f1)
03:20:54.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fa6b451e583 (/tmp/dist-test-taskgPZqXR/build/tsan/lib/libkudu_tools_util.so (deleted)+0x12e582)
03:20:54.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x6455a1 (/tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu (deleted)+0x6455a0)
03:20:54.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x645ed8 (/tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu (deleted)+0x645ed7)
03:20:54.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x64641f (/tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu (deleted)+0x64641e)
03:20:54.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7fa6a7f7fbf7 __libc_start_main
03:20:54.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x43399a (/tmp/dist-test-taskgPZqXR/build/tsan/bin/kudu (deleted)+0x433999)
03:20:55.014 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduClient.testStrings: failed attempt 1
java.io.EOFException: null
at java.io.DataInputStream.readInt(DataInputStream.java:392) ~[?:1.8.0_275]
at org.apache.kudu.test.cluster.MiniKuduCluster.sendRequestToCluster(MiniKuduCluster.java:191) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster.start(MiniKuduCluster.java:266) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster.access$300(MiniKuduCluster.java:85) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.cluster.MiniKuduCluster$MiniKuduClusterBuilder.build(MiniKuduCluster.java:883) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:156) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.io.EOFException
at java.io.DataInputStream.readInt(DataInputStream.java:392)
at org.apache.kudu.test.cluster.MiniKuduCluster.sendRequestToCluster(MiniKuduCluster.java:191)
at org.apache.kudu.test.cluster.MiniKuduCluster.start(MiniKuduCluster.java:266)
at org.apache.kudu.test.cluster.MiniKuduCluster.access$300(MiniKuduCluster.java:85)
at org.apache.kudu.test.cluster.MiniKuduCluster$MiniKuduClusterBuilder.build(MiniKuduCluster.java:883)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:156)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)