15:05:31.437 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
15:05:31.448 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskYvjSJN/test-tmp
15:05:31.450 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:05:31.453 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskYvjSJN/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:05:31.664 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
15:05:31.807 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:05:31.808 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:05:31.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:31.812664 8262 mini_oidc.cc:150] Starting JWKS server
15:05:31.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:31.816817 8262 webserver.cc:458] Webserver started at http://127.0.0.1:36795/ using document root <none> and password file <none>
15:05:31.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:31.817329 8262 mini_oidc.cc:186] Starting OIDC Discovery server
15:05:31.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:31.819273 8262 webserver.cc:458] Webserver started at http://127.0.0.1:45193/ using document root <none> and password file <none>
15:05:31.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-13T15:05:31Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
15:05:31.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-13T15:05:31Z Disabled control of system clock
15:05:31.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:31.877002 8262 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:31.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:31.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-0/wal
15:05:31.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-0/data
15:05:31.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:31.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:31.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:31.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:31.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:31.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:31.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-0/logs
15:05:31.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-0/data/info.pb
15:05:31.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:31.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:31.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:31.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:31.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:31.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:31.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:31.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:31.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:05:31.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:31.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:31.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:31.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.17.190:41133
15:05:31.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.17.190
15:05:31.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:31.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.17.148:40787
15:05:31.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:31.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:31.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:31.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:31.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.17.190:41133,127.8.17.189:39501,127.8.17.188:41027
15:05:31.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:31.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36795/jwks.json with env {}
15:05:32.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:32.155180 8274 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:32.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:32.155706 8274 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:32.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:32.156292 8274 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:32.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:32.184114 8274 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:05:32.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:32.184362 8274 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:32.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:32.184588 8274 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:05:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:32.184813 8274 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:05:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:32.185070 8274 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:05:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:32.185300 8274 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36795/jwks.json
15:05:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:32.223681 8274 master_runner.cc:386] Master server non-default flags:
15:05:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.17.148:40787
15:05:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-0/data
15:05:32.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-0/wal
15:05:32.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:32.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.17.190:41133,127.8.17.189:39501,127.8.17.188:41027
15:05:32.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:32.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:32.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:32.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.17.190:41133
15:05:32.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:32.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:32.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:32.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36795/jwks.json
15:05:32.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:32.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:32.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-0/data/info.pb
15:05:32.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.17.190
15:05:32.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:32.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:32.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:32.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:32.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:32.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:32.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-0/logs
15:05:32.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:32.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:32.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:32.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:05:32.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:32.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:32.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:32.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:05:32.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:05:32.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:32.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:32.225032 8274 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:32.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:32.226497 8274 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:32.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:32.240376 8281 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:32.240859 8280 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:32.242267 8274 server_base.cc:1034] running on GCE node
15:05:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:32.241456 8283 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:33.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.333148 8274 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:33.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.335702 8274 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:33.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.337033 8274 hybrid_clock.cc:648] HybridClock initialized: now 1736780733336992 us; error 51 us; skew 500 ppm
15:05:33.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.337808 8274 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:33.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.348618 8274 webserver.cc:458] Webserver started at http://127.8.17.190:41577/ using document root <none> and password file <none>
15:05:33.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.349577 8274 fs_manager.cc:362] Metadata directory not provided
15:05:33.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.349797 8274 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:33.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.350342 8274 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:33.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.354962 8274 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-0/data/instance:
15:05:33.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c4686c60c82e430486ebc08839f7cda4"
15:05:33.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:33 on dist-test-slave-s7w9"
15:05:33.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.356055 8274 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-0/wal/instance:
15:05:33.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c4686c60c82e430486ebc08839f7cda4"
15:05:33.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:33 on dist-test-slave-s7w9"
15:05:33.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.362398 8274 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.003s sys 0.005s
15:05:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.367148 8290 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:33.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.368047 8274 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
15:05:33.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.368353 8274 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-0/data,/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-0/wal
15:05:33.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c4686c60c82e430486ebc08839f7cda4"
15:05:33.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:33 on dist-test-slave-s7w9"
15:05:33.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.368709 8274 fs_report.cc:389] FS layout report
15:05:33.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:33.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-0/wal
15:05:33.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-0/wal
15:05:33.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-0/data/data
15:05:33.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:33.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:33.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:33.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:33.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:33.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:33.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:33.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:33.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:33.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:33.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:33.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.416081 8274 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:33.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.417480 8274 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:33.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.417970 8274 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:33.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.482584 8274 rpc_server.cc:307] RPC server started. Bound to: 127.8.17.190:41133
15:05:33.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.482671 8341 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.17.190:41133 every 8 connection(s)
15:05:33.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.485266 8274 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-0/data/info.pb
15:05:33.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.490090 8342 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:05:33.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.491314 8262 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu as pid 8274
15:05:33.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.491750 8262 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-0/wal/instance
15:05:33.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.494976 8262 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:33.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:33.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-1/wal
15:05:33.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-1/data
15:05:33.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:33.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-1/logs
15:05:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-1/data/info.pb
15:05:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:33.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:05:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:33.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:33.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.17.189:39501
15:05:33.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.17.189
15:05:33.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:33.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.17.148:40787
15:05:33.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:33.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:33.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:33.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:33.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.17.190:41133,127.8.17.189:39501,127.8.17.188:41027
15:05:33.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:33.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36795/jwks.json with env {}
15:05:33.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.501351 8342 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.17.190" port: 41133 } has no permanent_uuid. Determining permanent_uuid...
15:05:33.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.524029 8342 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } has no permanent_uuid. Determining permanent_uuid...
15:05:33.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:33.526278 8293 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.17.189:39501: connect: Connection refused (error 111)
15:05:33.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:33.530033 8342 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.17.189:39501: Network error: Client connection negotiation failed: client connection to 127.8.17.189:39501: connect: Connection refused (error 111)
15:05:33.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.579432 8342 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } attempt: 1
15:05:33.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:33.583631 8342 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.17.189:39501: Network error: Client connection negotiation failed: client connection to 127.8.17.189:39501: connect: Connection refused (error 111)
15:05:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.651989 8342 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } attempt: 2
15:05:33.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:33.655716 8342 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.17.189:39501: Network error: Client connection negotiation failed: client connection to 127.8.17.189:39501: connect: Connection refused (error 111)
15:05:33.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.747062 8342 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } attempt: 3
15:05:33.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:33.751152 8342 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.17.189:39501: Network error: Client connection negotiation failed: client connection to 127.8.17.189:39501: connect: Connection refused (error 111)
15:05:33.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:33.777855 8344 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:33.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:33.778527 8344 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:33.779172 8344 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:33.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:33.806723 8344 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:05:33.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:33.807016 8344 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:33.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:33.807334 8344 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:05:33.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:33.807618 8344 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:05:33.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:33.807919 8344 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:05:33.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:33.808132 8344 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36795/jwks.json
15:05:33.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.838337 8344 master_runner.cc:386] Master server non-default flags:
15:05:33.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:33.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.17.148:40787
15:05:33.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:33.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:33.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-1/data
15:05:33.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-1/wal
15:05:33.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:33.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.17.190:41133,127.8.17.189:39501,127.8.17.188:41027
15:05:33.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:33.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:33.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:33.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.17.189:39501
15:05:33.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:33.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:33.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:33.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36795/jwks.json
15:05:33.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:33.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:33.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-1/data/info.pb
15:05:33.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.17.189
15:05:33.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:33.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:33.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:33.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-1/logs
15:05:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:05:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:33.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:33.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:33.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:05:33.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:05:33.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:33.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.839949 8344 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:33.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.841884 8344 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:33.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:33.856195 8353 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:33.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.894513 8342 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } attempt: 4
15:05:33.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:33.899374 8342 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.17.189:39501: Network error: Client connection negotiation failed: client connection to 127.8.17.189:39501: connect: Connection refused (error 111)
15:05:34.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:34.198724 8342 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } attempt: 5
15:05:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:34.203296 8342 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.17.189:39501: Network error: Client connection negotiation failed: client connection to 127.8.17.189:39501: connect: Connection refused (error 111)
15:05:34.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:34.750644 8342 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } attempt: 6
15:05:34.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:34.763963 8342 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.17.189:39501: Network error: Client connection negotiation failed: client connection to 127.8.17.189:39501: connect: Connection refused (error 111)
15:05:34.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:33.856832 8356 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:34.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:33.857257 8354 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:34.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:33.857954 8344 server_base.cc:1034] running on GCE node
15:05:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:34.965744 8344 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:34.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:34.968643 8344 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:34.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:34.970052 8344 hybrid_clock.cc:648] HybridClock initialized: now 1736780734970012 us; error 56 us; skew 500 ppm
15:05:34.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:34.971030 8344 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:34.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:34.976542 8344 webserver.cc:458] Webserver started at http://127.8.17.189:32845/ using document root <none> and password file <none>
15:05:34.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:34.977429 8344 fs_manager.cc:362] Metadata directory not provided
15:05:34.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:34.977598 8344 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:34.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:34.978006 8344 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:34.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:34.982529 8344 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-1/data/instance:
15:05:34.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a53662348df24cf092eafc146b19373b"
15:05:34.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:34 on dist-test-slave-s7w9"
15:05:34.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:34.983587 8344 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-1/wal/instance:
15:05:34.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a53662348df24cf092eafc146b19373b"
15:05:34.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:34 on dist-test-slave-s7w9"
15:05:34.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:34.990178 8344 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.001s
15:05:34.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:34.995523 8365 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:34.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:34.996426 8344 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.003s
15:05:34.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:34.996711 8344 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-1/data,/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-1/wal
15:05:34.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a53662348df24cf092eafc146b19373b"
15:05:34.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:34 on dist-test-slave-s7w9"
15:05:34.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:34.997006 8344 fs_report.cc:389] FS layout report
15:05:34.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:34.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-1/wal
15:05:34.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-1/wal
15:05:34.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-1/data/data
15:05:34.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:34.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:34.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:34.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:35.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:35.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:35.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:35.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:35.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:35.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:35.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:35.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.041342 8344 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.042680 8344 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.043110 8344 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:35.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.107947 8344 rpc_server.cc:307] RPC server started. Bound to: 127.8.17.189:39501
15:05:35.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.108053 8416 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.17.189:39501 every 8 connection(s)
15:05:35.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.110735 8344 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-1/data/info.pb
15:05:35.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.115202 8417 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:05:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.121002 8262 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu as pid 8344
15:05:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.121433 8262 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-1/wal/instance
15:05:35.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.124382 8262 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:35.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:35.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-2/wal
15:05:35.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-2/data
15:05:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-2/logs
15:05:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-2/data/info.pb
15:05:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:05:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.17.188:41027
15:05:35.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.17.188
15:05:35.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:35.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.17.148:40787
15:05:35.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:35.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:35.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:35.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:35.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.17.190:41133,127.8.17.189:39501,127.8.17.188:41027
15:05:35.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:35.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36795/jwks.json with env {}
15:05:35.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.128427 8417 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.17.190" port: 41133 } has no permanent_uuid. Determining permanent_uuid...
15:05:35.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.151896 8417 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } has no permanent_uuid. Determining permanent_uuid...
15:05:35.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.164543 8417 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } has no permanent_uuid. Determining permanent_uuid...
15:05:35.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:35.166954 8369 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.17.188:41027: connect: Connection refused (error 111)
15:05:35.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:35.170840 8417 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.17.188:41027: Network error: Client connection negotiation failed: client connection to 127.8.17.188:41027: connect: Connection refused (error 111)
15:05:35.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.220202 8417 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } attempt: 1
15:05:35.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:35.224035 8417 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.17.188:41027: Network error: Client connection negotiation failed: client connection to 127.8.17.188:41027: connect: Connection refused (error 111)
15:05:35.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.292354 8417 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } attempt: 2
15:05:35.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:35.296398 8417 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.17.188:41027: Network error: Client connection negotiation failed: client connection to 127.8.17.188:41027: connect: Connection refused (error 111)
15:05:35.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.387801 8417 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } attempt: 3
15:05:35.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:35.391294 8417 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.17.188:41027: Network error: Client connection negotiation failed: client connection to 127.8.17.188:41027: connect: Connection refused (error 111)
15:05:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:35.411417 8419 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:35.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:35.411887 8419 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:35.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:35.412463 8419 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:35.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:35.440333 8419 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:05:35.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:35.440589 8419 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:35.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:35.440759 8419 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:05:35.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:35.441005 8419 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:05:35.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:35.441301 8419 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:05:35.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:35.441468 8419 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36795/jwks.json
15:05:35.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.472424 8419 master_runner.cc:386] Master server non-default flags:
15:05:35.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:35.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.17.148:40787
15:05:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-2/data
15:05:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-2/wal
15:05:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.17.190:41133,127.8.17.189:39501,127.8.17.188:41027
15:05:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.17.188:41027
15:05:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36795/jwks.json
15:05:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-2/data/info.pb
15:05:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.17.188
15:05:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:35.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:35.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:35.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:35.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-2/logs
15:05:35.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:35.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:35.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:35.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:05:35.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:35.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:35.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:35.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:05:35.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:05:35.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:35.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.473564 8419 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:35.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.475062 8419 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:35.491724 8429 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:35.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.534715 8417 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } attempt: 4
15:05:35.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:35.540335 8417 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.17.188:41027: Network error: Client connection negotiation failed: client connection to 127.8.17.188:41027: connect: Connection refused (error 111)
15:05:35.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.824606 8342 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } attempt: 7
15:05:35.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.840030 8417 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } attempt: 5
15:05:35.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:35.846210 8417 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.17.188:41027: Network error: Client connection negotiation failed: client connection to 127.8.17.188:41027: connect: Connection refused (error 111)
15:05:35.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.879882 8342 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } has no permanent_uuid. Determining permanent_uuid...
15:05:35.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:35.910053 8342 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.17.188:41027: Network error: Client connection negotiation failed: client connection to 127.8.17.188:41027: connect: Connection refused (error 111)
15:05:35.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:35.968580 8342 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } attempt: 1
15:05:35.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:35.974050 8342 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.17.188:41027: Network error: Client connection negotiation failed: client connection to 127.8.17.188:41027: connect: Connection refused (error 111)
15:05:36.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.055552 8342 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } attempt: 2
15:05:36.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:36.061311 8342 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.17.188:41027: Network error: Client connection negotiation failed: client connection to 127.8.17.188:41027: connect: Connection refused (error 111)
15:05:36.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.146842 8342 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } attempt: 3
15:05:36.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:36.161039 8342 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.17.188:41027: Network error: Client connection negotiation failed: client connection to 127.8.17.188:41027: connect: Connection refused (error 111)
15:05:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.301596 8342 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } attempt: 4
15:05:36.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:36.307390 8342 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.17.188:41027: Network error: Client connection negotiation failed: client connection to 127.8.17.188:41027: connect: Connection refused (error 111)
15:05:36.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.393776 8417 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } attempt: 6
15:05:36.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:36.403710 8417 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.17.188:41027: Network error: Client connection negotiation failed: client connection to 127.8.17.188:41027: connect: Connection refused (error 111)
15:05:36.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.591018 8342 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } attempt: 5
15:05:36.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:36.596220 8342 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.17.188:41027: Network error: Client connection negotiation failed: client connection to 127.8.17.188:41027: connect: Connection refused (error 111)
15:05:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:35.494585 8430 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:36.647984 8431 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
15:05:36.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:36.652927 8419 thread.cc:635] OpenStack (cloud detector) Time spent creating pthread: real 1.159s user 0.005s sys 0.000s
15:05:36.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:36.653285 8419 thread.cc:601] OpenStack (cloud detector) Time spent starting thread: real 1.160s user 0.005s sys 0.000s
15:05:36.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:36.654182 8432 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:36.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.654225 8419 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:05:36.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.655822 8419 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:36.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.659065 8419 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:36.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.660790 8419 hybrid_clock.cc:648] HybridClock initialized: now 1736780736660738 us; error 42 us; skew 500 ppm
15:05:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.662022 8419 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:36.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.671103 8419 webserver.cc:458] Webserver started at http://127.8.17.188:44549/ using document root <none> and password file <none>
15:05:36.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.674465 8419 fs_manager.cc:362] Metadata directory not provided
15:05:36.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.674687 8419 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:36.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.675170 8419 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:36.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.680213 8419 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-2/data/instance:
15:05:36.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d0d5eee999564d1db70a6483c48509d5"
15:05:36.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:36 on dist-test-slave-s7w9"
15:05:36.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.681581 8419 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-2/wal/instance:
15:05:36.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d0d5eee999564d1db70a6483c48509d5"
15:05:36.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:36 on dist-test-slave-s7w9"
15:05:36.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.692447 8419 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.001s sys 0.008s
15:05:36.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.697865 8445 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:36.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.698956 8419 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
15:05:36.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.699285 8419 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-2/data,/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-2/wal
15:05:36.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d0d5eee999564d1db70a6483c48509d5"
15:05:36.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:36 on dist-test-slave-s7w9"
15:05:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.699611 8419 fs_report.cc:389] FS layout report
15:05:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-2/wal
15:05:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-2/wal
15:05:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-2/data/data
15:05:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:36.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:36.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:36.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.763540 8419 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:36.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.765020 8419 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:36.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.765514 8419 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:36.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.828763 8419 rpc_server.cc:307] RPC server started. Bound to: 127.8.17.188:41027
15:05:36.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.828819 8496 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.17.188:41027 every 8 connection(s)
15:05:36.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.831473 8419 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-2/data/info.pb
15:05:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.836186 8497 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:05:36.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.836935 8262 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu as pid 8419
15:05:36.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.837371 8262 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/master-2/wal/instance
15:05:36.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.854765 8497 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.17.190" port: 41133 } has no permanent_uuid. Determining permanent_uuid...
15:05:36.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.877095 8497 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } has no permanent_uuid. Determining permanent_uuid...
15:05:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.886564 8497 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } has no permanent_uuid. Determining permanent_uuid...
15:05:36.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.900297 8497 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5: Bootstrap starting.
15:05:36.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.906405 8497 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5: Neither blocks nor log segments found. Creating new log.
15:05:36.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.908459 8497 log.cc:826] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5: Log is configured to *not* fsync() on all Append() calls
15:05:36.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.912611 8497 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5: No bootstrap required, opened a new log
15:05:36.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.932296 8497 raft_consensus.cc:357] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4686c60c82e430486ebc08839f7cda4" member_type: VOTER last_known_addr { host: "127.8.17.190" port: 41133 } } peers { permanent_uuid: "a53662348df24cf092eafc146b19373b" member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } } peers { permanent_uuid: "d0d5eee999564d1db70a6483c48509d5" member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } }
15:05:36.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.933382 8497 raft_consensus.cc:383] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:05:36.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.933665 8497 raft_consensus.cc:738] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d0d5eee999564d1db70a6483c48509d5, State: Initialized, Role: FOLLOWER
15:05:36.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.934473 8497 consensus_queue.cc:260] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4686c60c82e430486ebc08839f7cda4" member_type: VOTER last_known_addr { host: "127.8.17.190" port: 41133 } } peers { permanent_uuid: "a53662348df24cf092eafc146b19373b" member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } } peers { permanent_uuid: "d0d5eee999564d1db70a6483c48509d5" member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } }
15:05:36.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.936626 8506 sys_catalog.cc:455] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4686c60c82e430486ebc08839f7cda4" member_type: VOTER last_known_addr { host: "127.8.17.190" port: 41133 } } peers { permanent_uuid: "a53662348df24cf092eafc146b19373b" member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } } peers { permanent_uuid: "d0d5eee999564d1db70a6483c48509d5" member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } } }
15:05:36.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.937156 8506 sys_catalog.cc:458] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [sys.catalog]: This master's current role is: FOLLOWER
15:05:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.937952 8497 sys_catalog.cc:564] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [sys.catalog]: configured and running, proceeding with master startup.
15:05:36.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:36.959547 8517 catalog_manager.cc:1559] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:05:36.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:36.960206 8517 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:05:36.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.955566 8419 master_runner.cc:186] Error getting master registration for 127.8.17.190:41133: OK, instance_id { permanent_uuid: "c4686c60c82e430486ebc08839f7cda4" instance_seqno: 1736780733464773 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:05:36.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:36.978682 8419 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:05:37.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:37.020155 8451 tablet.cc:2367] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:05:37.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.130728 8506 raft_consensus.cc:491] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:05:37.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.131222 8506 raft_consensus.cc:513] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4686c60c82e430486ebc08839f7cda4" member_type: VOTER last_known_addr { host: "127.8.17.190" port: 41133 } } peers { permanent_uuid: "a53662348df24cf092eafc146b19373b" member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } } peers { permanent_uuid: "d0d5eee999564d1db70a6483c48509d5" member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } }
15:05:37.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.133255 8506 leader_election.cc:290] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c4686c60c82e430486ebc08839f7cda4 (127.8.17.190:41133), a53662348df24cf092eafc146b19373b (127.8.17.189:39501)
15:05:37.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.148624 8342 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } attempt: 6
15:05:37.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.168031 8342 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c4686c60c82e430486ebc08839f7cda4: Bootstrap starting.
15:05:37.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.174702 8342 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c4686c60c82e430486ebc08839f7cda4: Neither blocks nor log segments found. Creating new log.
15:05:37.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.176826 8342 log.cc:826] T 00000000000000000000000000000000 P c4686c60c82e430486ebc08839f7cda4: Log is configured to *not* fsync() on all Append() calls
15:05:37.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.183915 8342 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c4686c60c82e430486ebc08839f7cda4: No bootstrap required, opened a new log
15:05:37.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.212808 8342 raft_consensus.cc:357] T 00000000000000000000000000000000 P c4686c60c82e430486ebc08839f7cda4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4686c60c82e430486ebc08839f7cda4" member_type: VOTER last_known_addr { host: "127.8.17.190" port: 41133 } } peers { permanent_uuid: "a53662348df24cf092eafc146b19373b" member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } } peers { permanent_uuid: "d0d5eee999564d1db70a6483c48509d5" member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } }
15:05:37.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.213529 8342 raft_consensus.cc:383] T 00000000000000000000000000000000 P c4686c60c82e430486ebc08839f7cda4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:05:37.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.213840 8342 raft_consensus.cc:738] T 00000000000000000000000000000000 P c4686c60c82e430486ebc08839f7cda4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c4686c60c82e430486ebc08839f7cda4, State: Initialized, Role: FOLLOWER
15:05:37.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.214757 8342 consensus_queue.cc:260] T 00000000000000000000000000000000 P c4686c60c82e430486ebc08839f7cda4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4686c60c82e430486ebc08839f7cda4" member_type: VOTER last_known_addr { host: "127.8.17.190" port: 41133 } } peers { permanent_uuid: "a53662348df24cf092eafc146b19373b" member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } } peers { permanent_uuid: "d0d5eee999564d1db70a6483c48509d5" member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } }
15:05:37.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.218010 8521 sys_catalog.cc:455] T 00000000000000000000000000000000 P c4686c60c82e430486ebc08839f7cda4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4686c60c82e430486ebc08839f7cda4" member_type: VOTER last_known_addr { host: "127.8.17.190" port: 41133 } } peers { permanent_uuid: "a53662348df24cf092eafc146b19373b" member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } } peers { permanent_uuid: "d0d5eee999564d1db70a6483c48509d5" member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } } }
15:05:37.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.219061 8521 sys_catalog.cc:458] T 00000000000000000000000000000000 P c4686c60c82e430486ebc08839f7cda4 [sys.catalog]: This master's current role is: FOLLOWER
15:05:37.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.219980 8342 sys_catalog.cc:564] T 00000000000000000000000000000000 P c4686c60c82e430486ebc08839f7cda4 [sys.catalog]: configured and running, proceeding with master startup.
15:05:37.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.220167 8317 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d0d5eee999564d1db70a6483c48509d5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c4686c60c82e430486ebc08839f7cda4" is_pre_election: true
15:05:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.220963 8317 raft_consensus.cc:2463] T 00000000000000000000000000000000 P c4686c60c82e430486ebc08839f7cda4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d0d5eee999564d1db70a6483c48509d5 in term 0.
15:05:37.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.222136 8447 leader_election.cc:304] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c4686c60c82e430486ebc08839f7cda4, d0d5eee999564d1db70a6483c48509d5; no voters:
15:05:37.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.222959 8506 raft_consensus.cc:2798] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:05:37.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.223264 8506 raft_consensus.cc:491] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:05:37.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.223539 8506 raft_consensus.cc:3054] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [term 0 FOLLOWER]: Advancing to term 1
15:05:37.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.229496 8506 raft_consensus.cc:513] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4686c60c82e430486ebc08839f7cda4" member_type: VOTER last_known_addr { host: "127.8.17.190" port: 41133 } } peers { permanent_uuid: "a53662348df24cf092eafc146b19373b" member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } } peers { permanent_uuid: "d0d5eee999564d1db70a6483c48509d5" member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } }
15:05:37.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.231268 8506 leader_election.cc:290] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [CANDIDATE]: Term 1 election: Requested vote from peers c4686c60c82e430486ebc08839f7cda4 (127.8.17.190:41133), a53662348df24cf092eafc146b19373b (127.8.17.189:39501)
15:05:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.232211 8317 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d0d5eee999564d1db70a6483c48509d5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c4686c60c82e430486ebc08839f7cda4"
15:05:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.232776 8317 raft_consensus.cc:3054] T 00000000000000000000000000000000 P c4686c60c82e430486ebc08839f7cda4 [term 0 FOLLOWER]: Advancing to term 1
15:05:37.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.240187 8317 raft_consensus.cc:2463] T 00000000000000000000000000000000 P c4686c60c82e430486ebc08839f7cda4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d0d5eee999564d1db70a6483c48509d5 in term 1.
15:05:37.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.241683 8447 leader_election.cc:304] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c4686c60c82e430486ebc08839f7cda4, d0d5eee999564d1db70a6483c48509d5; no voters:
15:05:37.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.242831 8506 raft_consensus.cc:2798] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [term 1 FOLLOWER]: Leader election won for term 1
15:05:37.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.245550 8506 raft_consensus.cc:695] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [term 1 LEADER]: Becoming Leader. State: Replica: d0d5eee999564d1db70a6483c48509d5, State: Running, Role: LEADER
15:05:37.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.246524 8506 consensus_queue.cc:237] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4686c60c82e430486ebc08839f7cda4" member_type: VOTER last_known_addr { host: "127.8.17.190" port: 41133 } } peers { permanent_uuid: "a53662348df24cf092eafc146b19373b" member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } } peers { permanent_uuid: "d0d5eee999564d1db70a6483c48509d5" member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } }
15:05:37.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.267675 8525 sys_catalog.cc:455] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d0d5eee999564d1db70a6483c48509d5. Latest consensus state: current_term: 1 leader_uuid: "d0d5eee999564d1db70a6483c48509d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4686c60c82e430486ebc08839f7cda4" member_type: VOTER last_known_addr { host: "127.8.17.190" port: 41133 } } peers { permanent_uuid: "a53662348df24cf092eafc146b19373b" member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } } peers { permanent_uuid: "d0d5eee999564d1db70a6483c48509d5" member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } } }
15:05:37.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.268918 8525 sys_catalog.cc:458] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [sys.catalog]: This master's current role is: LEADER
15:05:37.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.276945 8529 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
15:05:37.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.285705 8529 catalog_manager.cc:1485] Initializing Kudu cluster ID...
15:05:37.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.310688 8317 raft_consensus.cc:1270] T 00000000000000000000000000000000 P c4686c60c82e430486ebc08839f7cda4 [term 1 FOLLOWER]: Refusing update from remote peer d0d5eee999564d1db70a6483c48509d5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:05:37.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.312505 8525 consensus_queue.cc:1035] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c4686c60c82e430486ebc08839f7cda4" member_type: VOTER last_known_addr { host: "127.8.17.190" port: 41133 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:05:37.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:37.330858 8535 catalog_manager.cc:1559] T 00000000000000000000000000000000 P c4686c60c82e430486ebc08839f7cda4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:05:37.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:37.331136 8535 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:05:37.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.307292 8274 master_runner.cc:186] Error getting master registration for 127.8.17.189:39501: OK, instance_id { permanent_uuid: "a53662348df24cf092eafc146b19373b" instance_seqno: 1736780735090410 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:05:37.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.343005 8274 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:05:37.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.356622 8521 sys_catalog.cc:455] T 00000000000000000000000000000000 P c4686c60c82e430486ebc08839f7cda4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d0d5eee999564d1db70a6483c48509d5. Latest consensus state: current_term: 1 leader_uuid: "d0d5eee999564d1db70a6483c48509d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4686c60c82e430486ebc08839f7cda4" member_type: VOTER last_known_addr { host: "127.8.17.190" port: 41133 } } peers { permanent_uuid: "a53662348df24cf092eafc146b19373b" member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } } peers { permanent_uuid: "d0d5eee999564d1db70a6483c48509d5" member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } } }
15:05:37.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.357283 8521 sys_catalog.cc:458] T 00000000000000000000000000000000 P c4686c60c82e430486ebc08839f7cda4 [sys.catalog]: This master's current role is: FOLLOWER
15:05:37.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.365658 8536 mvcc.cc:204] Tried to move back new op lower bound from 7113853900005806080 to 7113853899800641536. Current Snapshot: MvccSnapshot[applied={T|T < 7113853900005806080}]
15:05:37.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.367198 8506 sys_catalog.cc:455] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d0d5eee999564d1db70a6483c48509d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4686c60c82e430486ebc08839f7cda4" member_type: VOTER last_known_addr { host: "127.8.17.190" port: 41133 } } peers { permanent_uuid: "a53662348df24cf092eafc146b19373b" member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } } peers { permanent_uuid: "d0d5eee999564d1db70a6483c48509d5" member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } } }
15:05:37.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.368157 8506 sys_catalog.cc:458] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [sys.catalog]: This master's current role is: LEADER
15:05:37.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.371798 8529 catalog_manager.cc:1348] Generated new cluster ID: c714f07f37ec41019248481dd739e626
15:05:37.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.371629 8521 sys_catalog.cc:455] T 00000000000000000000000000000000 P c4686c60c82e430486ebc08839f7cda4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d0d5eee999564d1db70a6483c48509d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4686c60c82e430486ebc08839f7cda4" member_type: VOTER last_known_addr { host: "127.8.17.190" port: 41133 } } peers { permanent_uuid: "a53662348df24cf092eafc146b19373b" member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } } peers { permanent_uuid: "d0d5eee999564d1db70a6483c48509d5" member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } } }
15:05:37.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.372085 8529 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
15:05:37.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.372295 8521 sys_catalog.cc:458] T 00000000000000000000000000000000 P c4686c60c82e430486ebc08839f7cda4 [sys.catalog]: This master's current role is: FOLLOWER
15:05:37.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.406149 8529 catalog_manager.cc:1371] Generated new certificate authority record
15:05:37.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.408694 8529 catalog_manager.cc:1505] Loading token signing keys...
15:05:37.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.428051 8529 catalog_manager.cc:5899] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5: Generated new TSK 0
15:05:37.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.430666 8529 catalog_manager.cc:1515] Initializing in-progress tserver states...
15:05:37.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.441960 8262 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:37.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:37.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-0/wal
15:05:37.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-0/data
15:05:37.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:37.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:37.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:37.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:37.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:37.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:37.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-0/logs
15:05:37.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-0/data/info.pb
15:05:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:05:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.17.129:0
15:05:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.17.129
15:05:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.17.129
15:05:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.17.190:41133,127.8.17.189:39501,127.8.17.188:41027
15:05:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.17.148:40787
15:05:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:05:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.464185 8417 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } attempt: 7
15:05:37.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.494733 8417 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a53662348df24cf092eafc146b19373b: Bootstrap starting.
15:05:37.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.504004 8417 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a53662348df24cf092eafc146b19373b: Neither blocks nor log segments found. Creating new log.
15:05:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.508805 8417 log.cc:826] T 00000000000000000000000000000000 P a53662348df24cf092eafc146b19373b: Log is configured to *not* fsync() on all Append() calls
15:05:37.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.528417 8417 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a53662348df24cf092eafc146b19373b: No bootstrap required, opened a new log
15:05:37.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.580339 8417 raft_consensus.cc:357] T 00000000000000000000000000000000 P a53662348df24cf092eafc146b19373b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4686c60c82e430486ebc08839f7cda4" member_type: VOTER last_known_addr { host: "127.8.17.190" port: 41133 } } peers { permanent_uuid: "a53662348df24cf092eafc146b19373b" member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } } peers { permanent_uuid: "d0d5eee999564d1db70a6483c48509d5" member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } }
15:05:37.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.581919 8417 raft_consensus.cc:383] T 00000000000000000000000000000000 P a53662348df24cf092eafc146b19373b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:05:37.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.582417 8417 raft_consensus.cc:738] T 00000000000000000000000000000000 P a53662348df24cf092eafc146b19373b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a53662348df24cf092eafc146b19373b, State: Initialized, Role: FOLLOWER
15:05:37.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.583890 8417 consensus_queue.cc:260] T 00000000000000000000000000000000 P a53662348df24cf092eafc146b19373b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4686c60c82e430486ebc08839f7cda4" member_type: VOTER last_known_addr { host: "127.8.17.190" port: 41133 } } peers { permanent_uuid: "a53662348df24cf092eafc146b19373b" member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } } peers { permanent_uuid: "d0d5eee999564d1db70a6483c48509d5" member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } }
15:05:37.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.615104 8545 sys_catalog.cc:455] T 00000000000000000000000000000000 P a53662348df24cf092eafc146b19373b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4686c60c82e430486ebc08839f7cda4" member_type: VOTER last_known_addr { host: "127.8.17.190" port: 41133 } } peers { permanent_uuid: "a53662348df24cf092eafc146b19373b" member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } } peers { permanent_uuid: "d0d5eee999564d1db70a6483c48509d5" member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } } }
15:05:37.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.615948 8545 sys_catalog.cc:458] T 00000000000000000000000000000000 P a53662348df24cf092eafc146b19373b [sys.catalog]: This master's current role is: FOLLOWER
15:05:37.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.619432 8391 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d0d5eee999564d1db70a6483c48509d5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a53662348df24cf092eafc146b19373b"
15:05:37.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.619472 8392 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d0d5eee999564d1db70a6483c48509d5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a53662348df24cf092eafc146b19373b" is_pre_election: true
15:05:37.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.620251 8391 raft_consensus.cc:3054] T 00000000000000000000000000000000 P a53662348df24cf092eafc146b19373b [term 0 FOLLOWER]: Advancing to term 1
15:05:37.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.626937 8391 raft_consensus.cc:2463] T 00000000000000000000000000000000 P a53662348df24cf092eafc146b19373b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d0d5eee999564d1db70a6483c48509d5 in term 1.
15:05:37.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.628158 8390 raft_consensus.cc:1270] T 00000000000000000000000000000000 P a53662348df24cf092eafc146b19373b [term 1 FOLLOWER]: Refusing update from remote peer d0d5eee999564d1db70a6483c48509d5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:05:37.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.635092 8506 consensus_queue.cc:1035] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a53662348df24cf092eafc146b19373b" member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:05:37.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.642638 8417 sys_catalog.cc:564] T 00000000000000000000000000000000 P a53662348df24cf092eafc146b19373b [sys.catalog]: configured and running, proceeding with master startup.
15:05:37.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.717105 8545 sys_catalog.cc:455] T 00000000000000000000000000000000 P a53662348df24cf092eafc146b19373b [sys.catalog]: SysCatalogTable state changed. Reason: New leader d0d5eee999564d1db70a6483c48509d5. Latest consensus state: current_term: 1 leader_uuid: "d0d5eee999564d1db70a6483c48509d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4686c60c82e430486ebc08839f7cda4" member_type: VOTER last_known_addr { host: "127.8.17.190" port: 41133 } } peers { permanent_uuid: "a53662348df24cf092eafc146b19373b" member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } } peers { permanent_uuid: "d0d5eee999564d1db70a6483c48509d5" member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } } }
15:05:37.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.717216 8551 sys_catalog.cc:455] T 00000000000000000000000000000000 P a53662348df24cf092eafc146b19373b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d0d5eee999564d1db70a6483c48509d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4686c60c82e430486ebc08839f7cda4" member_type: VOTER last_known_addr { host: "127.8.17.190" port: 41133 } } peers { permanent_uuid: "a53662348df24cf092eafc146b19373b" member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } } peers { permanent_uuid: "d0d5eee999564d1db70a6483c48509d5" member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } } }
15:05:37.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.717900 8545 sys_catalog.cc:458] T 00000000000000000000000000000000 P a53662348df24cf092eafc146b19373b [sys.catalog]: This master's current role is: FOLLOWER
15:05:37.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.717988 8551 sys_catalog.cc:458] T 00000000000000000000000000000000 P a53662348df24cf092eafc146b19373b [sys.catalog]: This master's current role is: FOLLOWER
15:05:37.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:37.762916 8562 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:05:37.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:37.952680 8542 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:37.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:37.953154 8542 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:37.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:37.953693 8542 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:37.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:37.984052 8542 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:37.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:37.984922 8542 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.17.129
15:05:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.017208 8542 tablet_server_runner.cc:78] Tablet server non-default flags:
15:05:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.17.148:40787
15:05:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-0/data
15:05:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-0/wal
15:05:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.17.129:0
15:05:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-0/data/info.pb
15:05:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.17.129
15:05:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.17.190:41133,127.8.17.189:39501,127.8.17.188:41027
15:05:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.17.129
15:05:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-0/logs
15:05:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:05:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:05:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:05:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.018817 8542 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.020620 8542 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:38.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.047039 8569 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:38.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.096582 8506 sys_catalog.cc:455] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d0d5eee999564d1db70a6483c48509d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4686c60c82e430486ebc08839f7cda4" member_type: VOTER last_known_addr { host: "127.8.17.190" port: 41133 } } peers { permanent_uuid: "a53662348df24cf092eafc146b19373b" member_type: VOTER last_known_addr { host: "127.8.17.189" port: 39501 } } peers { permanent_uuid: "d0d5eee999564d1db70a6483c48509d5" member_type: VOTER last_known_addr { host: "127.8.17.188" port: 41027 } } }
15:05:38.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.098014 8506 sys_catalog.cc:458] T 00000000000000000000000000000000 P d0d5eee999564d1db70a6483c48509d5 [sys.catalog]: This master's current role is: LEADER
15:05:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.336648 8535 catalog_manager.cc:1260] Loaded cluster ID: c714f07f37ec41019248481dd739e626
15:05:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.336896 8535 catalog_manager.cc:1553] T 00000000000000000000000000000000 P c4686c60c82e430486ebc08839f7cda4: loading cluster ID for follower catalog manager: success
15:05:38.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.343681 8535 catalog_manager.cc:1575] T 00000000000000000000000000000000 P c4686c60c82e430486ebc08839f7cda4: acquiring CA information for follower catalog manager: success
15:05:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.349002 8535 catalog_manager.cc:1603] T 00000000000000000000000000000000 P c4686c60c82e430486ebc08839f7cda4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:05:38.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.788998 8562 catalog_manager.cc:1260] Loaded cluster ID: c714f07f37ec41019248481dd739e626
15:05:38.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.789861 8562 catalog_manager.cc:1553] T 00000000000000000000000000000000 P a53662348df24cf092eafc146b19373b: loading cluster ID for follower catalog manager: success
15:05:38.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.807113 8562 catalog_manager.cc:1575] T 00000000000000000000000000000000 P a53662348df24cf092eafc146b19373b: acquiring CA information for follower catalog manager: success
15:05:38.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.837976 8562 catalog_manager.cc:1603] T 00000000000000000000000000000000 P a53662348df24cf092eafc146b19373b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:05:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.056664 8571 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:38.057185 8542 server_base.cc:1034] running on GCE node
15:05:39.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:38.055512 8568 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:39.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.403331 8542 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:39.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.406102 8542 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:39.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.407557 8542 hybrid_clock.cc:648] HybridClock initialized: now 1736780739407571 us; error 142 us; skew 500 ppm
15:05:39.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.408695 8542 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:39.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.420429 8542 webserver.cc:458] Webserver started at http://127.8.17.129:36667/ using document root <none> and password file <none>
15:05:39.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.421640 8542 fs_manager.cc:362] Metadata directory not provided
15:05:39.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.421904 8542 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:39.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.422473 8542 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:39.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.429376 8542 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-0/data/instance:
15:05:39.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6bb8cf810fd94ee1926093cb2a2be069"
15:05:39.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:39 on dist-test-slave-s7w9"
15:05:39.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.430895 8542 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-0/wal/instance:
15:05:39.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6bb8cf810fd94ee1926093cb2a2be069"
15:05:39.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:39 on dist-test-slave-s7w9"
15:05:39.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.440176 8542 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.005s sys 0.003s
15:05:39.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.447783 8581 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:39.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.449051 8542 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
15:05:39.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.449424 8542 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-0/data,/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-0/wal
15:05:39.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6bb8cf810fd94ee1926093cb2a2be069"
15:05:39.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:39 on dist-test-slave-s7w9"
15:05:39.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.449864 8542 fs_report.cc:389] FS layout report
15:05:39.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:39.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-0/wal
15:05:39.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-0/wal
15:05:39.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-0/data/data
15:05:39.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:39.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:39.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:39.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:39.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:39.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:39.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:39.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:39.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:39.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:39.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:39.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.531738 8542 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:39.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.533561 8542 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:39.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.534073 8542 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:39.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.538427 8542 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:05:39.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.544152 8542 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:05:39.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.544450 8542 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:05:39.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.544762 8542 ts_tablet_manager.cc:610] Registered 0 tablets
15:05:39.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.544988 8542 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:05:39.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.717247 8542 rpc_server.cc:307] RPC server started. Bound to: 127.8.17.129:34293
15:05:39.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.717373 8693 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.17.129:34293 every 8 connection(s)
15:05:39.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.720822 8542 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-0/data/info.pb
15:05:39.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.728040 8262 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu as pid 8542
15:05:39.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.728475 8262 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-0/wal/instance
15:05:39.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.755460 8262 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:39.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:39.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-1/wal
15:05:39.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-1/data
15:05:39.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:39.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:39.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:39.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:39.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:39.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:39.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-1/logs
15:05:39.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-1/data/info.pb
15:05:39.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:39.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:39.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:39.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:39.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:39.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:39.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:39.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:39.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:05:39.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:39.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.17.130:0
15:05:39.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.17.130
15:05:39.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.17.130
15:05:39.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:39.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.17.190:41133,127.8.17.189:39501,127.8.17.188:41027
15:05:39.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.17.148:40787
15:05:39.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:39.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:39.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:05:39.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.832249 8695 heartbeater.cc:346] Connected to a master server at 127.8.17.189:39501
15:05:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.838361 8695 heartbeater.cc:463] Registering TS with master...
15:05:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.839519 8695 heartbeater.cc:510] Master 127.8.17.189:39501 requested a full tablet report, sending...
15:05:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.835348 8694 heartbeater.cc:346] Connected to a master server at 127.8.17.188:41027
15:05:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.840775 8694 heartbeater.cc:463] Registering TS with master...
15:05:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.841544 8694 heartbeater.cc:510] Master 127.8.17.188:41027 requested a full tablet report, sending...
15:05:39.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.843935 8462 ts_manager.cc:194] Registered new tserver with Master: 6bb8cf810fd94ee1926093cb2a2be069 (127.8.17.129:34293)
15:05:39.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.848145 8382 ts_manager.cc:194] Registered new tserver with Master: 6bb8cf810fd94ee1926093cb2a2be069 (127.8.17.129:34293)
15:05:39.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.850003 8462 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.17.129:54353
15:05:39.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.856379 8699 heartbeater.cc:346] Connected to a master server at 127.8.17.190:41133
15:05:39.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.856657 8699 heartbeater.cc:463] Registering TS with master...
15:05:39.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.857241 8699 heartbeater.cc:510] Master 127.8.17.190:41133 requested a full tablet report, sending...
15:05:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:39.859762 8307 ts_manager.cc:194] Registered new tserver with Master: 6bb8cf810fd94ee1926093cb2a2be069 (127.8.17.129:34293)
15:05:40.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.141153 8698 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:40.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.141734 8698 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:40.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.142434 8698 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:40.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.174392 8698 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:40.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.175184 8698 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.17.130
15:05:40.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.207515 8698 tablet_server_runner.cc:78] Tablet server non-default flags:
15:05:40.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:40.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.17.148:40787
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-1/data
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-1/wal
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.17.130:0
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-1/data/info.pb
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.17.130
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.17.190:41133,127.8.17.189:39501,127.8.17.188:41027
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.17.130
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-1/logs
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:05:40.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:40.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:40.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:40.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:05:40.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:05:40.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:40.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.208979 8698 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:40.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.210929 8698 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.231072 8711 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:40.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.853942 8694 heartbeater.cc:502] Master 127.8.17.188:41027 was elected leader, sending a full tablet report...
15:05:41.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.231698 8713 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:41.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:40.232765 8698 server_base.cc:1034] running on GCE node
15:05:41.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:40.232210 8710 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:41.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.353353 8698 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:41.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.355332 8698 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:41.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.356666 8698 hybrid_clock.cc:648] HybridClock initialized: now 1736780741356646 us; error 41 us; skew 500 ppm
15:05:41.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.357357 8698 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:41.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.362955 8698 webserver.cc:458] Webserver started at http://127.8.17.130:44059/ using document root <none> and password file <none>
15:05:41.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.363724 8698 fs_manager.cc:362] Metadata directory not provided
15:05:41.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.363886 8698 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:41.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.364217 8698 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:41.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.368594 8698 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-1/data/instance:
15:05:41.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2cf616adc7fa41c98d81fff6e765b13d"
15:05:41.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:41 on dist-test-slave-s7w9"
15:05:41.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.369627 8698 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-1/wal/instance:
15:05:41.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2cf616adc7fa41c98d81fff6e765b13d"
15:05:41.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:41 on dist-test-slave-s7w9"
15:05:41.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.375828 8698 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.001s sys 0.004s
15:05:41.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.380467 8720 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.381382 8698 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
15:05:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.381639 8698 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-1/data,/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-1/wal
15:05:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2cf616adc7fa41c98d81fff6e765b13d"
15:05:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:41 on dist-test-slave-s7w9"
15:05:41.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.381887 8698 fs_report.cc:389] FS layout report
15:05:41.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:41.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-1/wal
15:05:41.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-1/wal
15:05:41.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-1/data/data
15:05:41.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:41.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:41.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:41.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:41.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:41.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:41.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:41.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:41.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:41.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:41.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:41.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.422605 8698 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:41.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.424100 8698 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:41.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.424511 8698 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:41.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.426898 8698 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:05:41.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.430676 8698 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:05:41.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.430873 8698 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:05:41.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.431077 8698 ts_tablet_manager.cc:610] Registered 0 tablets
15:05:41.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.431218 8698 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:05:41.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.557680 8698 rpc_server.cc:307] RPC server started. Bound to: 127.8.17.130:36139
15:05:41.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.557782 8832 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.17.130:36139 every 8 connection(s)
15:05:41.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.559952 8698 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-1/data/info.pb
15:05:41.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.567808 8262 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu as pid 8698
15:05:41.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.568333 8262 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-1/wal/instance
15:05:41.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.598680 8262 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:41.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:41.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-2/wal
15:05:41.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-2/data
15:05:41.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:41.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:41.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:41.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:41.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:41.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-2/logs
15:05:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-2/data/info.pb
15:05:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:05:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.17.131:0
15:05:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.17.131
15:05:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.17.131
15:05:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.17.190:41133,127.8.17.189:39501,127.8.17.188:41027
15:05:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.17.148:40787
15:05:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:05:41.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.604918 8833 heartbeater.cc:346] Connected to a master server at 127.8.17.188:41027
15:05:41.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.605381 8833 heartbeater.cc:463] Registering TS with master...
15:05:41.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.606662 8833 heartbeater.cc:510] Master 127.8.17.188:41027 requested a full tablet report, sending...
15:05:41.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.606853 8835 heartbeater.cc:346] Connected to a master server at 127.8.17.190:41133
15:05:41.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.607132 8835 heartbeater.cc:463] Registering TS with master...
15:05:41.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.608247 8835 heartbeater.cc:510] Master 127.8.17.190:41133 requested a full tablet report, sending...
15:05:41.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.609009 8462 ts_manager.cc:194] Registered new tserver with Master: 2cf616adc7fa41c98d81fff6e765b13d (127.8.17.130:36139)
15:05:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.610126 8834 heartbeater.cc:346] Connected to a master server at 127.8.17.189:39501
15:05:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.610445 8834 heartbeater.cc:463] Registering TS with master...
15:05:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.610685 8462 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.17.130:50123
15:05:41.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.611230 8834 heartbeater.cc:510] Master 127.8.17.189:39501 requested a full tablet report, sending...
15:05:41.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.612555 8307 ts_manager.cc:194] Registered new tserver with Master: 2cf616adc7fa41c98d81fff6e765b13d (127.8.17.130:36139)
15:05:41.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.613565 8382 ts_manager.cc:194] Registered new tserver with Master: 2cf616adc7fa41c98d81fff6e765b13d (127.8.17.130:36139)
15:05:41.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.890009 8843 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:41.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.890491 8843 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:41.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.891043 8843 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:41.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.919062 8843 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:41.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.919890 8843 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.17.131
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.950863 8843 tablet_server_runner.cc:78] Tablet server non-default flags:
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.17.148:40787
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-2/data
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-2/wal
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.17.131:0
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-2/data/info.pb
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.17.131
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.17.190:41133,127.8.17.189:39501,127.8.17.188:41027
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.17.131
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-2/logs
15:05:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:05:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:05:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:05:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.952355 8843 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:41.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.953701 8843 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:41.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.967792 8850 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:42.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:42.614398 8833 heartbeater.cc:502] Master 127.8.17.188:41027 was elected leader, sending a full tablet report...
15:05:43.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.968206 8851 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:43.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:41.970511 8843 server_base.cc:1034] running on GCE node
15:05:43.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:41.969583 8853 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:43.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.064146 8843 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:43.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.066651 8843 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:43.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.068030 8843 hybrid_clock.cc:648] HybridClock initialized: now 1736780743067995 us; error 53 us; skew 500 ppm
15:05:43.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.068718 8843 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.075160 8843 webserver.cc:458] Webserver started at http://127.8.17.131:32913/ using document root <none> and password file <none>
15:05:43.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.075920 8843 fs_manager.cc:362] Metadata directory not provided
15:05:43.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.076108 8843 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:43.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.076467 8843 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.080904 8843 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-2/data/instance:
15:05:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4f2788683f77434d938761bf73553357"
15:05:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:43 on dist-test-slave-s7w9"
15:05:43.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.081835 8843 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-2/wal/instance:
15:05:43.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4f2788683f77434d938761bf73553357"
15:05:43.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:43 on dist-test-slave-s7w9"
15:05:43.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.087941 8843 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
15:05:43.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.093056 8861 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:43.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.094097 8843 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
15:05:43.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.094394 8843 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-2/data,/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-2/wal
15:05:43.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4f2788683f77434d938761bf73553357"
15:05:43.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:43 on dist-test-slave-s7w9"
15:05:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.094691 8843 fs_report.cc:389] FS layout report
15:05:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-2/wal
15:05:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-2/wal
15:05:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-2/data/data
15:05:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:43.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:43.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.148277 8843 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:43.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.149560 8843 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:43.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.149941 8843 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:43.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.152245 8843 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:05:43.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.156066 8843 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:05:43.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.156267 8843 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:05:43.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.156453 8843 ts_tablet_manager.cc:610] Registered 0 tablets
15:05:43.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.156580 8843 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:05:43.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.277892 8843 rpc_server.cc:307] RPC server started. Bound to: 127.8.17.131:45081
15:05:43.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.278013 8973 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.17.131:45081 every 8 connection(s)
15:05:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.280226 8843 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-2/data/info.pb
15:05:43.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.290706 8262 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu as pid 8843
15:05:43.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.291469 8262 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster5605466461775807637/ts-2/wal/instance
15:05:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.329778 8974 heartbeater.cc:346] Connected to a master server at 127.8.17.188:41027
15:05:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.330041 8976 heartbeater.cc:346] Connected to a master server at 127.8.17.190:41133
15:05:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.330284 8974 heartbeater.cc:463] Registering TS with master...
15:05:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.330452 8976 heartbeater.cc:463] Registering TS with master...
15:05:43.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.331277 8975 heartbeater.cc:346] Connected to a master server at 127.8.17.189:39501
15:05:43.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.331585 8975 heartbeater.cc:463] Registering TS with master...
15:05:43.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.331648 8974 heartbeater.cc:510] Master 127.8.17.188:41027 requested a full tablet report, sending...
15:05:43.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.331620 8976 heartbeater.cc:510] Master 127.8.17.190:41133 requested a full tablet report, sending...
15:05:43.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.332262 8975 heartbeater.cc:510] Master 127.8.17.189:39501 requested a full tablet report, sending...
15:05:43.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.333859 8462 ts_manager.cc:194] Registered new tserver with Master: 4f2788683f77434d938761bf73553357 (127.8.17.131:45081)
15:05:43.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.334275 8307 ts_manager.cc:194] Registered new tserver with Master: 4f2788683f77434d938761bf73553357 (127.8.17.131:45081)
15:05:43.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.334705 8382 ts_manager.cc:194] Registered new tserver with Master: 4f2788683f77434d938761bf73553357 (127.8.17.131:45081)
15:05:43.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.335306 8462 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.17.131:58195
15:05:43.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.347499 8262 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
15:05:43.357 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:05:43.358 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
15:05:43.367 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.8.17.190"
port: 41133
}
bound_http_address {
host: "127.8.17.190"
port: 41577
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.8.17.189"
port: 39501
}
bound_http_address {
host: "127.8.17.189"
port: 32845
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.8.17.188"
port: 41027
}
bound_http_address {
host: "127.8.17.188"
port: 44549
}
}
}
15:05:43.370 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:05:43.372 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.8.17.129"
port: 34293
}
bound_http_address {
host: "127.8.17.129"
port: 36667
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.8.17.130"
port: 36139
}
bound_http_address {
host: "127.8.17.130"
port: 44059
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.8.17.131"
port: 45081
}
bound_http_address {
host: "127.8.17.131"
port: 32913
}
}
}
15:05:43.372 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
15:05:43.573 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
15:05:43.645 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskYvjSJN/test-tmp
15:05:43.645 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:05:43.646 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskYvjSJN/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:05:43.651 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
15:05:43.952 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:05:43.952 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:05:43.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.955276 8985 mini_oidc.cc:150] Starting JWKS server
15:05:43.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.958424 8985 webserver.cc:458] Webserver started at http://127.0.0.1:41287/ using document root <none> and password file <none>
15:05:43.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.958837 8985 mini_oidc.cc:186] Starting OIDC Discovery server
15:05:43.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:43.960404 8985 webserver.cc:458] Webserver started at http://127.0.0.1:36013/ using document root <none> and password file <none>
15:05:44.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
15:05:44.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/principal' for realm 'KRBTEST.COM',
15:05:44.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
15:05:44.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:44 dist-test-slave-s7w9 krb5kdc[8998](info): setting up network...
15:05:44.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
15:05:44.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:44 dist-test-slave-s7w9 krb5kdc[8998](info): set up 2 sockets
15:05:44.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:44 dist-test-slave-s7w9 krb5kdc[8998](info): commencing operation
15:05:44.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
15:05:44.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:44.337642 8974 heartbeater.cc:502] Master 127.8.17.188:41027 was elected leader, sending a full tablet report...
15:05:44.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:44.990895 8985 mini_kdc.cc:121] Time spent starting KDC: real 0.995s user 0.006s sys 0.011s
15:05:45.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
15:05:45.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:05:45.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
15:05:45.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
15:05:45.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:05:45.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
15:05:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
15:05:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:05:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
15:05:45.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:05:45.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/test-user.keytab.
15:05:45.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/test-user.keytab.
15:05:45.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:45 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780745, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:05:45.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:05:45.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-13T15:05:45Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
15:05:45.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-13T15:05:45Z Disabled control of system clock
15:05:45.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.198.126@KRBTEST.COM; defaulting to no policy
15:05:45.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:05:45.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.198.126@KRBTEST.COM" created.
15:05:45.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:05:45.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.198.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.126.keytab.
15:05:45.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.198.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.126.keytab.
15:05:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.153875 8985 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/wal
15:05:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data
15:05:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/logs
15:05:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data/info.pb
15:05:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.126:39143
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.126
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.126.keytab
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.126
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:05:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:05:45.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.439838 9011 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:45.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.440372 9011 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:45.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.440943 9011 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:45.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.471509 9011 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:05:45.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.471756 9011 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:05:45.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.472024 9011 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:45.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.472239 9011 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:05:45.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.472430 9011 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:05:45.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.472597 9011 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:05:45.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.472726 9011 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41287/jwks.json
15:05:45.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.509553 9011 master_runner.cc:386] Master server non-default flags:
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/wal
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.126.keytab
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.126
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.126:39143
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data/info.pb
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.126
15:05:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/logs
15:05:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:05:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:05:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:05:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.510931 9011 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.511854 9011 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:45.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.527086 9018 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.527603 9019 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:45.528053 9021 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:45.528705 9011 server_base.cc:1034] running on GCE node
15:05:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.667394 9011 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:46.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.669641 9011 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:46.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.671205 9011 hybrid_clock.cc:648] HybridClock initialized: now 1736780746671144 us; error 53 us; skew 500 ppm
15:05:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.671989 9011 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:46.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:46 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780746, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:05:46.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.676951 9011 init.cc:377] Logged in from keytab as kudu/127.8.198.126@KRBTEST.COM (short username kudu)
15:05:46.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.681558 9011 webserver.cc:458] Webserver started at http://127.8.198.126:44225/ using document root <none> and password file <none>
15:05:46.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.682379 9011 fs_manager.cc:362] Metadata directory not provided
15:05:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.682569 9011 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.683018 9011 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:46.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.687768 9011 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data/instance:
15:05:46.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "251861861371402797d330f9ecdd6143"
15:05:46.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:46 on dist-test-slave-s7w9"
15:05:46.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.688850 9011 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/wal/instance:
15:05:46.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "251861861371402797d330f9ecdd6143"
15:05:46.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:46 on dist-test-slave-s7w9"
15:05:46.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.695444 9011 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.004s sys 0.004s
15:05:46.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.700153 9029 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:46.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.701182 9011 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
15:05:46.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.701495 9011 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data,/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/wal
15:05:46.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "251861861371402797d330f9ecdd6143"
15:05:46.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:46 on dist-test-slave-s7w9"
15:05:46.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.701776 9011 fs_report.cc:389] FS layout report
15:05:46.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:46.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/wal
15:05:46.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/wal
15:05:46.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data/data
15:05:46.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:46.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:46.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:46.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:46.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:46.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:46.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:46.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:46.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:46.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:46.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:46.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.752708 9011 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:46.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.757076 9011 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:46.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.757409 9011 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:46.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.819170 9011 rpc_server.cc:307] RPC server started. Bound to: 127.8.198.126:39143
15:05:46.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.819239 9080 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.198.126:39143 every 8 connection(s)
15:05:46.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.821692 9011 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data/info.pb
15:05:46.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.826174 8985 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu as pid 9011
15:05:46.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.826675 8985 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/wal/instance
15:05:46.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.827417 9081 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:05:46.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.841236 9081 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } has no permanent_uuid. Determining permanent_uuid...
15:05:46.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.198.125@KRBTEST.COM; defaulting to no policy
15:05:46.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:05:46.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.198.125@KRBTEST.COM" created.
15:05:46.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:46 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780746, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.126@KRBTEST.COM for kudu/127.8.198.126@KRBTEST.COM
15:05:46.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:05:46.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.198.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.125.keytab.
15:05:46.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.198.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.125.keytab.
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.874857 8985 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/wal
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/logs
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data/info.pb
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.125:33167
15:05:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.125
15:05:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.125.keytab
15:05:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.125
15:05:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:05:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:05:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:05:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:05:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:05:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:05:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:05:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.875530 9081 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } has no permanent_uuid. Determining permanent_uuid...
15:05:46.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:46.877849 9032 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111)
15:05:46.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:46.881796 9081 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.198.125:33167: Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111)
15:05:46.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:46.931365 9081 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } attempt: 1
15:05:46.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:46.934814 9081 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.198.125:33167: Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111)
15:05:47.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.003214 9081 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } attempt: 2
15:05:47.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.006878 9081 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.198.125:33167: Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111)
15:05:47.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.098309 9081 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } attempt: 3
15:05:47.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.102970 9081 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.198.125:33167: Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111)
15:05:47.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.155342 9087 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:47.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.155807 9087 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:47.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.156330 9087 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:47.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.183848 9087 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:05:47.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.184147 9087 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:05:47.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.184422 9087 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:47.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.184604 9087 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:05:47.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.184777 9087 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:05:47.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.185050 9087 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:05:47.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.185318 9087 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41287/jwks.json
15:05:47.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.218983 9087 master_runner.cc:386] Master server non-default flags:
15:05:47.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:47.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:05:47.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:47.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:47.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data
15:05:47.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/wal
15:05:47.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:05:47.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:47.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:05:47.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.125.keytab
15:05:47.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.125
15:05:47.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:47.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:47.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:47.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.125:33167
15:05:47.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:47.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:47.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data/info.pb
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.125
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/logs
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.220585 9087 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:47.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.221601 9087 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:47.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.236244 9094 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:47.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.246346 9081 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } attempt: 4
15:05:47.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.249683 9081 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.198.125:33167: Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111)
15:05:47.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.549039 9081 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } attempt: 5
15:05:47.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.553299 9081 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.198.125:33167: Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111)
15:05:48.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.100644 9081 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } attempt: 6
15:05:48.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:48.106640 9081 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.198.125:33167: Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111)
15:05:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.236816 9095 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:47.237290 9097 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:47.238258 9087 server_base.cc:1034] running on GCE node
15:05:48.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.384896 9087 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:48.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.387326 9087 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:48.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.388631 9087 hybrid_clock.cc:648] HybridClock initialized: now 1736780748388597 us; error 45 us; skew 500 ppm
15:05:48.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.389317 9087 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:48.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:48 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780748, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:05:48.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.394769 9087 init.cc:377] Logged in from keytab as kudu/127.8.198.125@KRBTEST.COM (short username kudu)
15:05:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.399437 9087 webserver.cc:458] Webserver started at http://127.8.198.125:37437/ using document root <none> and password file <none>
15:05:48.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.400586 9087 fs_manager.cc:362] Metadata directory not provided
15:05:48.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.400851 9087 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:48.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.401398 9087 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:48.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.407967 9087 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data/instance:
15:05:48.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "712570e4ca7d40e9b588643a35fd6197"
15:05:48.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:48 on dist-test-slave-s7w9"
15:05:48.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.409075 9087 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/wal/instance:
15:05:48.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "712570e4ca7d40e9b588643a35fd6197"
15:05:48.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:48 on dist-test-slave-s7w9"
15:05:48.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.415647 9087 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.009s sys 0.000s
15:05:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.420571 9108 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:48.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.421650 9087 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
15:05:48.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.421887 9087 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data,/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/wal
15:05:48.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "712570e4ca7d40e9b588643a35fd6197"
15:05:48.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:48 on dist-test-slave-s7w9"
15:05:48.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.422192 9087 fs_report.cc:389] FS layout report
15:05:48.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:48.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/wal
15:05:48.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/wal
15:05:48.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data/data
15:05:48.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:48.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:48.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:48.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:48.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:48.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:48.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:48.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:48.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:48.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:48.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:48.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.483525 9087 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.490203 9087 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.490711 9087 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:48.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.563203 9087 rpc_server.cc:307] RPC server started. Bound to: 127.8.198.125:33167
15:05:48.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.563252 9159 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.198.125:33167 every 8 connection(s)
15:05:48.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.566723 9087 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data/info.pb
15:05:48.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.569234 8985 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu as pid 9087
15:05:48.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.569703 8985 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/wal/instance
15:05:48.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.573176 9160 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:05:48.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.198.124@KRBTEST.COM; defaulting to no policy
15:05:48.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.584714 9160 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } has no permanent_uuid. Determining permanent_uuid...
15:05:48.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:05:48.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.198.124@KRBTEST.COM" created.
15:05:48.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:48 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780748, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.125@KRBTEST.COM for kudu/127.8.198.126@KRBTEST.COM
15:05:48.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:05:48.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.198.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.124.keytab.
15:05:48.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.198.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.124.keytab.
15:05:48.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.612298 8985 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:48.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:48.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/wal
15:05:48.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data
15:05:48.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:48.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:48.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:48.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/logs
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data/info.pb
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.124:34423
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.124
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.124.keytab
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.124
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:48.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:48.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:48.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:05:48.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:48.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:05:48.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:05:48.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.618907 9160 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } has no permanent_uuid. Determining permanent_uuid...
15:05:48.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:48 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780748, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.125@KRBTEST.COM for kudu/127.8.198.125@KRBTEST.COM
15:05:48.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.642714 9160 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } has no permanent_uuid. Determining permanent_uuid...
15:05:48.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:48.645087 9109 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.198.124:34423: connect: Connection refused (error 111)
15:05:48.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:48.648974 9160 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.198.124:34423: Network error: Client connection negotiation failed: client connection to 127.8.198.124:34423: connect: Connection refused (error 111)
15:05:48.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.698486 9160 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } attempt: 1
15:05:48.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:48.702080 9160 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.198.124:34423: Network error: Client connection negotiation failed: client connection to 127.8.198.124:34423: connect: Connection refused (error 111)
15:05:48.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.770591 9160 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } attempt: 2
15:05:48.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:48.773900 9160 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.198.124:34423: Network error: Client connection negotiation failed: client connection to 127.8.198.124:34423: connect: Connection refused (error 111)
15:05:48.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.865350 9160 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } attempt: 3
15:05:48.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:48.868729 9160 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.198.124:34423: Network error: Client connection negotiation failed: client connection to 127.8.198.124:34423: connect: Connection refused (error 111)
15:05:48.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:48.907008 9166 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:48.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:48.907567 9166 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:48.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:48.908154 9166 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:48.938565 9166 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:05:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:48.938781 9166 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:05:48.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:48.939069 9166 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:48.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:48.939313 9166 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:05:48.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:48.939500 9166 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:05:48.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:48.939677 9166 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:05:48.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:48.939813 9166 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41287/jwks.json
15:05:48.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.976940 9166 master_runner.cc:386] Master server non-default flags:
15:05:48.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:48.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:05:48.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:48.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:48.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data
15:05:48.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/wal
15:05:48.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:05:48.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:48.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:05:48.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.124.keytab
15:05:48.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.124
15:05:48.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:48.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:48.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:48.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.124:34423
15:05:48.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:48.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:48.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:48.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:05:48.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data/info.pb
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.124
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/logs
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:48.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.978263 9166 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:48.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.979192 9166 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:48.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:48.992552 9175 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:49.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:49.012133 9160 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } attempt: 4
15:05:49.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:49.015889 9160 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.198.124:34423: Network error: Client connection negotiation failed: client connection to 127.8.198.124:34423: connect: Connection refused (error 111)
15:05:49.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:49.167186 9081 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } attempt: 7
15:05:49.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:49 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780746, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.126@KRBTEST.COM for kudu/127.8.198.125@KRBTEST.COM
15:05:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:49.189705 9081 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } has no permanent_uuid. Determining permanent_uuid...
15:05:49.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:49.195842 9081 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.198.124:34423: Network error: Client connection negotiation failed: client connection to 127.8.198.124:34423: connect: Connection refused (error 111)
15:05:49.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:49.254176 9081 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } attempt: 1
15:05:49.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:49.258654 9081 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.198.124:34423: Network error: Client connection negotiation failed: client connection to 127.8.198.124:34423: connect: Connection refused (error 111)
15:05:49.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:49.315466 9160 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } attempt: 5
15:05:49.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:49.320389 9160 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.198.124:34423: Network error: Client connection negotiation failed: client connection to 127.8.198.124:34423: connect: Connection refused (error 111)
15:05:49.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:49.339978 9081 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } attempt: 2
15:05:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:49.344339 9081 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.198.124:34423: Network error: Client connection negotiation failed: client connection to 127.8.198.124:34423: connect: Connection refused (error 111)
15:05:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:49.429687 9081 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } attempt: 3
15:05:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:49.434630 9081 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.198.124:34423: Network error: Client connection negotiation failed: client connection to 127.8.198.124:34423: connect: Connection refused (error 111)
15:05:49.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:49.574986 9081 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } attempt: 4
15:05:49.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:49.580129 9081 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.198.124:34423: Network error: Client connection negotiation failed: client connection to 127.8.198.124:34423: connect: Connection refused (error 111)
15:05:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:49.863567 9081 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } attempt: 5
15:05:49.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:49.867764 9160 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } attempt: 6
15:05:49.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:49.868324 9081 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.198.124:34423: Network error: Client connection negotiation failed: client connection to 127.8.198.124:34423: connect: Connection refused (error 111)
15:05:49.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:49.873875 9160 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.198.124:34423: Network error: Client connection negotiation failed: client connection to 127.8.198.124:34423: connect: Connection refused (error 111)
15:05:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:48.995659 9166 server_base.cc:1034] running on GCE node
15:05:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:48.993702 9177 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:48.993309 9174 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.173125 9166 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:50.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.175562 9166 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:50.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.176862 9166 hybrid_clock.cc:648] HybridClock initialized: now 1736780750176839 us; error 39 us; skew 500 ppm
15:05:50.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.177597 9166 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:50.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:50 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780750, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:05:50.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.183020 9166 init.cc:377] Logged in from keytab as kudu/127.8.198.124@KRBTEST.COM (short username kudu)
15:05:50.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.187283 9166 webserver.cc:458] Webserver started at http://127.8.198.124:36391/ using document root <none> and password file <none>
15:05:50.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.188158 9166 fs_manager.cc:362] Metadata directory not provided
15:05:50.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.188360 9166 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:50.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.188805 9166 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:50.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.193324 9166 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data/instance:
15:05:50.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a32bb1830f4f4f458a1ad99a5ed3122f"
15:05:50.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:50 on dist-test-slave-s7w9"
15:05:50.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.194397 9166 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/wal/instance:
15:05:50.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a32bb1830f4f4f458a1ad99a5ed3122f"
15:05:50.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:50 on dist-test-slave-s7w9"
15:05:50.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.201195 9166 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
15:05:50.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.206280 9194 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:50.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.207267 9166 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
15:05:50.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.207530 9166 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data,/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/wal
15:05:50.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a32bb1830f4f4f458a1ad99a5ed3122f"
15:05:50.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:50 on dist-test-slave-s7w9"
15:05:50.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.207798 9166 fs_report.cc:389] FS layout report
15:05:50.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:50.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/wal
15:05:50.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/wal
15:05:50.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data/data
15:05:50.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:50.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:50.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:50.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:50.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:50.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:50.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:50.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:50.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:50.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:50.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:50.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.255926 9166 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:50.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.260826 9166 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:50.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.261294 9166 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:50.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.336372 9166 rpc_server.cc:307] RPC server started. Bound to: 127.8.198.124:34423
15:05:50.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.336448 9245 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.198.124:34423 every 8 connection(s)
15:05:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.339072 9166 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data/info.pb
15:05:50.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.346645 9246 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:05:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.348531 8985 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu as pid 9166
15:05:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.348986 8985 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/wal/instance
15:05:50.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.362733 9246 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } has no permanent_uuid. Determining permanent_uuid...
15:05:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:50 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780745, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.198.126@KRBTEST.COM
15:05:50.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:50 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780750, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.124@KRBTEST.COM for kudu/127.8.198.126@KRBTEST.COM
15:05:50.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.400210 9246 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } has no permanent_uuid. Determining permanent_uuid...
15:05:50.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:50 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780745, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.198.125@KRBTEST.COM
15:05:50.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:50 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780750, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.124@KRBTEST.COM for kudu/127.8.198.125@KRBTEST.COM
15:05:50.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.420768 9081 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } attempt: 6
15:05:50.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.421135 9246 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } has no permanent_uuid. Determining permanent_uuid...
15:05:50.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:50 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780745, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.198.124@KRBTEST.COM
15:05:50.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:50 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780746, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.126@KRBTEST.COM for kudu/127.8.198.124@KRBTEST.COM
15:05:50.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:50 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780750, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.124@KRBTEST.COM for kudu/127.8.198.124@KRBTEST.COM
15:05:50.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.457082 9081 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: Bootstrap starting.
15:05:50.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.458894 9246 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: Bootstrap starting.
15:05:50.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.464179 9081 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: Neither blocks nor log segments found. Creating new log.
15:05:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.465709 9246 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: Neither blocks nor log segments found. Creating new log.
15:05:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.466324 9081 log.cc:826] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: Log is configured to *not* fsync() on all Append() calls
15:05:50.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.467952 9246 log.cc:826] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: Log is configured to *not* fsync() on all Append() calls
15:05:50.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.471278 9081 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: No bootstrap required, opened a new log
15:05:50.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.473192 9246 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: No bootstrap required, opened a new log
15:05:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.491891 9246 raft_consensus.cc:357] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:05:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.492466 9246 raft_consensus.cc:383] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:05:50.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.492722 9246 raft_consensus.cc:738] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a32bb1830f4f4f458a1ad99a5ed3122f, State: Initialized, Role: FOLLOWER
15:05:50.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.492982 9081 raft_consensus.cc:357] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:05:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.493784 9081 raft_consensus.cc:383] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:05:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.493579 9246 consensus_queue.cc:260] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:05:50.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.494171 9081 raft_consensus.cc:738] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 251861861371402797d330f9ecdd6143, State: Initialized, Role: FOLLOWER
15:05:50.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.496073 9081 consensus_queue.cc:260] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:05:50.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.501003 9261 sys_catalog.cc:455] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:05:50.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.502203 9261 sys_catalog.cc:458] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: This master's current role is: FOLLOWER
15:05:50.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.502509 9081 sys_catalog.cc:564] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: configured and running, proceeding with master startup.
15:05:50.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.503650 9260 sys_catalog.cc:455] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:05:50.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.504415 9260 sys_catalog.cc:458] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: This master's current role is: FOLLOWER
15:05:50.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.505223 9246 sys_catalog.cc:564] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: configured and running, proceeding with master startup.
15:05:50.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:50.514691 9035 tablet.cc:2367] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:05:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:50.516222 9200 tablet.cc:2367] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:05:50.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:50.553655 9281 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:05:50.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:50.554042 9281 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:05:50.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.542424 9011 master_runner.cc:186] Error getting master registration for 127.8.198.125:33167: OK, instance_id { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" instance_seqno: 1736780748544509 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:05:50.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.572175 9011 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:05:50.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.578516 9260 raft_consensus.cc:491] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:05:50.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.580456 9260 raft_consensus.cc:513] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:05:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.584018 9260 leader_election.cc:290] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 251861861371402797d330f9ecdd6143 (127.8.198.126:39143), 712570e4ca7d40e9b588643a35fd6197 (127.8.198.125:33167)
15:05:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.589277 9056 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "251861861371402797d330f9ecdd6143" is_pre_election: true
15:05:50.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.590098 9056 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a32bb1830f4f4f458a1ad99a5ed3122f in term 0.
15:05:50.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.591600 9197 leader_election.cc:304] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 251861861371402797d330f9ecdd6143, a32bb1830f4f4f458a1ad99a5ed3122f; no voters:
15:05:50.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.593238 9260 raft_consensus.cc:2798] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 0 FOLLOWER]: Leader pre-election won for term 1
15:05:50.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.593612 9260 raft_consensus.cc:491] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:05:50.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.593940 9260 raft_consensus.cc:3054] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 0 FOLLOWER]: Advancing to term 1
15:05:50.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:50.595948 9283 catalog_manager.cc:1559] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:05:50.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:50.596690 9283 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:05:50.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.605633 9260 raft_consensus.cc:513] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:05:50.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.549437 9166 master_runner.cc:186] Error getting master registration for 127.8.198.125:33167: OK, instance_id { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" instance_seqno: 1736780748544509 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:05:50.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.616062 9166 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:05:50.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.616026 9056 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "251861861371402797d330f9ecdd6143"
15:05:50.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.616534 9056 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 0 FOLLOWER]: Advancing to term 1
15:05:50.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.618717 9260 leader_election.cc:290] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [CANDIDATE]: Term 1 election: Requested vote from peers 251861861371402797d330f9ecdd6143 (127.8.198.126:39143), 712570e4ca7d40e9b588643a35fd6197 (127.8.198.125:33167)
15:05:50.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.622706 9056 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a32bb1830f4f4f458a1ad99a5ed3122f in term 1.
15:05:50.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.623580 9197 leader_election.cc:304] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 251861861371402797d330f9ecdd6143, a32bb1830f4f4f458a1ad99a5ed3122f; no voters:
15:05:50.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.624454 9260 raft_consensus.cc:2798] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 1 FOLLOWER]: Leader election won for term 1
15:05:50.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.630893 9260 raft_consensus.cc:695] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 1 LEADER]: Becoming Leader. State: Replica: a32bb1830f4f4f458a1ad99a5ed3122f, State: Running, Role: LEADER
15:05:50.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.632726 9260 consensus_queue.cc:237] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:05:50.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.645471 9286 sys_catalog.cc:455] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: SysCatalogTable state changed. Reason: New leader a32bb1830f4f4f458a1ad99a5ed3122f. Latest consensus state: current_term: 1 leader_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:05:50.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.650355 9286 sys_catalog.cc:458] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: This master's current role is: LEADER
15:05:50.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.654127 9288 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
15:05:50.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.661284 9288 catalog_manager.cc:1485] Initializing Kudu cluster ID...
15:05:50.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.684125 9056 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 1 FOLLOWER]: Refusing update from remote peer a32bb1830f4f4f458a1ad99a5ed3122f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:05:50.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.686682 9260 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [LEADER]: Connected to new peer: Peer: permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:05:50.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.721961 9261 sys_catalog.cc:455] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a32bb1830f4f4f458a1ad99a5ed3122f. Latest consensus state: current_term: 1 leader_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:05:50.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.723598 9261 sys_catalog.cc:458] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: This master's current role is: FOLLOWER
15:05:50.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.735082 9261 sys_catalog.cc:455] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:05:50.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.735771 9261 sys_catalog.cc:458] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: This master's current role is: FOLLOWER
15:05:50.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.739440 9286 sys_catalog.cc:455] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:05:50.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.740135 9286 sys_catalog.cc:458] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: This master's current role is: LEADER
15:05:50.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.742869 9288 catalog_manager.cc:1348] Generated new cluster ID: ef52103ef7db46f78a1ee3713871c7d3
15:05:50.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.743147 9288 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
15:05:50.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.784736 9288 catalog_manager.cc:1371] Generated new certificate authority record
15:05:50.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.788586 9288 catalog_manager.cc:1505] Loading token signing keys...
15:05:50.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.819017 9288 catalog_manager.cc:5899] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: Generated new TSK 0
15:05:50.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.819912 9288 catalog_manager.cc:1515] Initializing in-progress tserver states...
15:05:50.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.198.65@KRBTEST.COM; defaulting to no policy
15:05:50.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:05:50.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.198.65@KRBTEST.COM" created.
15:05:50.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:05:50.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.198.65 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.65.keytab.
15:05:50.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.198.65 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.65.keytab.
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.933509 8985 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-0/wal
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-0/data
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-0/logs
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-0/data/info.pb
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.65:0
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.198.65
15:05:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.65
15:05:50.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:50.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:05:50.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.65.keytab
15:05:50.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.65
15:05:50.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:05:50.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:05:50.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:05:50.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:05:50.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:50.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:50.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {KRB5CCNAME=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:05:50.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:50.934417 9160 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } attempt: 7
15:05:50.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:50 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780748, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.125@KRBTEST.COM for kudu/127.8.198.124@KRBTEST.COM
15:05:51.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.020911 9160 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: Bootstrap starting.
15:05:51.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.037783 9160 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: Neither blocks nor log segments found. Creating new log.
15:05:51.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.040112 9160 log.cc:826] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: Log is configured to *not* fsync() on all Append() calls
15:05:51.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.059600 9160 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: No bootstrap required, opened a new log
15:05:51.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.181790 9160 raft_consensus.cc:357] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:05:51.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.187397 9160 raft_consensus.cc:383] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:05:51.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.189436 9160 raft_consensus.cc:738] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 712570e4ca7d40e9b588643a35fd6197, State: Initialized, Role: FOLLOWER
15:05:51.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.194959 9160 consensus_queue.cc:260] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:05:51.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.209467 9299 sys_catalog.cc:455] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:05:51.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.210263 9299 sys_catalog.cc:458] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: This master's current role is: FOLLOWER
15:05:51.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.218282 9133 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [term 0 FOLLOWER]: Advancing to term 1
15:05:51.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.221832 9135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "712570e4ca7d40e9b588643a35fd6197" is_pre_election: true
15:05:51.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.223376 9134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "712570e4ca7d40e9b588643a35fd6197"
15:05:51.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.232273 9133 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [term 1 FOLLOWER]: Refusing update from remote peer a32bb1830f4f4f458a1ad99a5ed3122f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:05:51.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.234037 9260 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [LEADER]: Connected to new peer: Peer: permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:05:51.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:51.248174 9114 tablet.cc:2367] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:05:51.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.250094 9160 sys_catalog.cc:564] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: configured and running, proceeding with master startup.
15:05:51.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.331094 9299 sys_catalog.cc:455] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a32bb1830f4f4f458a1ad99a5ed3122f. Latest consensus state: current_term: 1 leader_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:05:51.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.331262 9305 sys_catalog.cc:455] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:05:51.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.332166 9299 sys_catalog.cc:458] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: This master's current role is: FOLLOWER
15:05:51.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.332345 9305 sys_catalog.cc:458] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: This master's current role is: FOLLOWER
15:05:51.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.422145 9315 catalog_manager.cc:1260] Loaded cluster ID: ef52103ef7db46f78a1ee3713871c7d3
15:05:51.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.422403 9315 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: loading cluster ID for follower catalog manager: success
15:05:51.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.442633 9315 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: acquiring CA information for follower catalog manager: success
15:05:51.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.464277 9315 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:05:51.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.566816 9281 catalog_manager.cc:1260] Loaded cluster ID: ef52103ef7db46f78a1ee3713871c7d3
15:05:51.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.567314 9281 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: loading cluster ID for follower catalog manager: success
15:05:51.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.584348 9281 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: acquiring CA information for follower catalog manager: success
15:05:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.589334 9281 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:05:51.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:51.625924 9297 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:51.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:51.626618 9297 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:51.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:51.627519 9297 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:51.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:51.689787 9297 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:51.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:51.692499 9297 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.198.65
15:05:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.755879 9297 tablet_server_runner.cc:78] Tablet server non-default flags:
15:05:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:05:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-0/data
15:05:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-0/wal
15:05:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.65.keytab
15:05:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.65
15:05:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.65:0
15:05:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:05:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:05:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:05:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-0/data/info.pb
15:05:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.65
15:05:51.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:51.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:05:51.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:51.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:51.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:51.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:51.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:51.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.198.65
15:05:51.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-0/logs
15:05:51.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:51.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:05:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:05:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:05:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.757495 9297 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:51.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.758839 9297 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:51.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:51.799170 9323 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:51.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.868161 9326 sys_catalog.cc:455] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:05:51.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:51.868849 9326 sys_catalog.cc:458] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: This master's current role is: LEADER
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:53.182328 9322 debug-util.cc:398] Leaking SignalData structure 0x7b080001ca40 after lost signal to thread 9297
15:05:54.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:51.801482 9324 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:54.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:54.076197 9297 thread.cc:635] GCE (cloud detector) Time spent creating pthread: real 2.278s user 0.423s sys 0.804s
15:05:54.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:54.076767 9297 thread.cc:601] GCE (cloud detector) Time spent starting thread: real 2.279s user 0.423s sys 0.804s
15:05:54.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.086580 9297 server_base.cc:1034] running on GCE node
15:05:54.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:54.087283 9337 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:54.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.088583 9297 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:54.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.090919 9297 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:54.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.092293 9297 hybrid_clock.cc:648] HybridClock initialized: now 1736780754092248 us; error 32 us; skew 500 ppm
15:05:54.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.093184 9297 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:54.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:54 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780754, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:05:54.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.099407 9297 init.cc:377] Logged in from keytab as kudu/127.8.198.65@KRBTEST.COM (short username kudu)
15:05:54.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.104351 9297 webserver.cc:458] Webserver started at http://127.8.198.65:41201/ using document root <none> and password file <none>
15:05:54.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.105253 9297 fs_manager.cc:362] Metadata directory not provided
15:05:54.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.105548 9297 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:54.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.105903 9297 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:54.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.110797 9297 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-0/data/instance:
15:05:54.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4af75b1b5ad4450385a38dc956ee3969"
15:05:54.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:54 on dist-test-slave-s7w9"
15:05:54.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.112164 9297 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-0/wal/instance:
15:05:54.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4af75b1b5ad4450385a38dc956ee3969"
15:05:54.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:54 on dist-test-slave-s7w9"
15:05:54.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.120019 9297 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.003s sys 0.003s
15:05:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.126232 9343 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.127399 9297 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
15:05:54.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.127714 9297 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-0/data,/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-0/wal
15:05:54.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4af75b1b5ad4450385a38dc956ee3969"
15:05:54.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:54 on dist-test-slave-s7w9"
15:05:54.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.128053 9297 fs_report.cc:389] FS layout report
15:05:54.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:54.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-0/wal
15:05:54.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-0/wal
15:05:54.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-0/data/data
15:05:54.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:54.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:54.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:54.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:54.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:54.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:54.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:54.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:54.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:54.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:54.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:54.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.187592 9297 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:54.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.193166 9297 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:54.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.193624 9297 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:54.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.196141 9297 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:05:54.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.200568 9297 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:05:54.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.200793 9297 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:05:54.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.201090 9297 ts_tablet_manager.cc:610] Registered 0 tablets
15:05:54.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.201262 9297 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:05:54.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.391589 9297 rpc_server.cc:307] RPC server started. Bound to: 127.8.198.65:42545
15:05:54.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.391734 9455 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.198.65:42545 every 8 connection(s)
15:05:54.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.394201 9297 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-0/data/info.pb
15:05:54.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.403611 8985 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu as pid 9297
15:05:54.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.404325 8985 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-0/wal/instance
15:05:54.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:54 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780754, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.65@KRBTEST.COM for kudu/127.8.198.125@KRBTEST.COM
15:05:54.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:54 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780754, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.65@KRBTEST.COM for kudu/127.8.198.124@KRBTEST.COM
15:05:54.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.198.66@KRBTEST.COM; defaulting to no policy
15:05:54.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:05:54.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.198.66@KRBTEST.COM" created.
15:05:54.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:54 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780754, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.65@KRBTEST.COM for kudu/127.8.198.126@KRBTEST.COM
15:05:54.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.487596 9456 heartbeater.cc:346] Connected to a master server at 127.8.198.124:34423
15:05:54.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.488044 9456 heartbeater.cc:463] Registering TS with master...
15:05:54.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.489240 9456 heartbeater.cc:510] Master 127.8.198.124:34423 requested a full tablet report, sending...
15:05:54.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.490147 9457 heartbeater.cc:346] Connected to a master server at 127.8.198.125:33167
15:05:54.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.490475 9457 heartbeater.cc:463] Registering TS with master...
15:05:54.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.491410 9457 heartbeater.cc:510] Master 127.8.198.125:33167 requested a full tablet report, sending...
15:05:54.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.494870 9125 ts_manager.cc:194] Registered new tserver with Master: 4af75b1b5ad4450385a38dc956ee3969 (127.8.198.65:42545)
15:05:54.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.495270 9211 ts_manager.cc:194] Registered new tserver with Master: 4af75b1b5ad4450385a38dc956ee3969 (127.8.198.65:42545)
15:05:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.497527 9211 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.8.198.65@KRBTEST.COM'} at 127.8.198.65:41657
15:05:54.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.498988 9458 heartbeater.cc:346] Connected to a master server at 127.8.198.126:39143
15:05:54.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.499311 9458 heartbeater.cc:463] Registering TS with master...
15:05:54.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.500154 9458 heartbeater.cc:510] Master 127.8.198.126:39143 requested a full tablet report, sending...
15:05:54.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.502970 9046 ts_manager.cc:194] Registered new tserver with Master: 4af75b1b5ad4450385a38dc956ee3969 (127.8.198.65:42545)
15:05:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:05:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.198.66 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.66.keytab.
15:05:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.198.66 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.66.keytab.
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.522588 8985 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-1/wal
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-1/data
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-1/logs
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-1/data/info.pb
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.66:0
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.198.66
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.66
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.66.keytab
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.66
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {KRB5CCNAME=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:05:54.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:54.847499 9468 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:54.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:54.848066 9468 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:54.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:54.848925 9468 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:54.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:54.879743 9468 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:54.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:54.880694 9468 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.198.66
15:05:54.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.916152 9468 tablet_server_runner.cc:78] Tablet server non-default flags:
15:05:54.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:54.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:05:54.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:54.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:54.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-1/data
15:05:54.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-1/wal
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.66.keytab
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.66
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.66:0
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-1/data/info.pb
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.66
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.198.66
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-1/logs
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:54.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:54.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:05:54.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:05:54.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:54.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.917690 9468 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:54.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.919032 9468 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:54.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:54.937384 9474 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:55.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:55.502523 9456 heartbeater.cc:502] Master 127.8.198.124:34423 was elected leader, sending a full tablet report...
15:05:56.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:54.937701 9477 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:54.938441 9475 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:54.939237 9468 server_base.cc:1034] running on GCE node
15:05:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.117628 9468 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:56.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.120332 9468 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:56.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.121765 9468 hybrid_clock.cc:648] HybridClock initialized: now 1736780756121731 us; error 31 us; skew 500 ppm
15:05:56.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.122773 9468 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:56.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:56 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780756, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:05:56.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.135892 9468 init.cc:377] Logged in from keytab as kudu/127.8.198.66@KRBTEST.COM (short username kudu)
15:05:56.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.141106 9468 webserver.cc:458] Webserver started at http://127.8.198.66:39599/ using document root <none> and password file <none>
15:05:56.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.141973 9468 fs_manager.cc:362] Metadata directory not provided
15:05:56.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.142207 9468 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:56.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.142575 9468 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:56.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.147238 9468 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-1/data/instance:
15:05:56.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "314efce02ef6446094dd961b666f16b4"
15:05:56.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:56 on dist-test-slave-s7w9"
15:05:56.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.148200 9468 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-1/wal/instance:
15:05:56.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "314efce02ef6446094dd961b666f16b4"
15:05:56.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:56 on dist-test-slave-s7w9"
15:05:56.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.155484 9468 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
15:05:56.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.160647 9488 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:56.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.161636 9468 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
15:05:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.161976 9468 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-1/data,/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-1/wal
15:05:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "314efce02ef6446094dd961b666f16b4"
15:05:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:56 on dist-test-slave-s7w9"
15:05:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.162336 9468 fs_report.cc:389] FS layout report
15:05:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-1/wal
15:05:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-1/wal
15:05:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-1/data/data
15:05:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:56.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.231348 9468 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:56.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.236332 9468 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:56.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.236790 9468 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:56.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.239287 9468 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:05:56.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.243156 9468 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:05:56.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.243350 9468 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:05:56.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.243651 9468 ts_tablet_manager.cc:610] Registered 0 tablets
15:05:56.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.243815 9468 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:05:56.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.403149 9468 rpc_server.cc:307] RPC server started. Bound to: 127.8.198.66:37253
15:05:56.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.403182 9600 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.198.66:37253 every 8 connection(s)
15:05:56.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.406628 9468 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-1/data/info.pb
15:05:56.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.410885 8985 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu as pid 9468
15:05:56.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.411366 8985 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-1/wal/instance
15:05:56.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.8.198.67@KRBTEST.COM; defaulting to no policy
15:05:56.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:05:56.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.8.198.67@KRBTEST.COM" created.
15:05:56.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:56 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780756, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.66@KRBTEST.COM for kudu/127.8.198.125@KRBTEST.COM
15:05:56.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:56 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780756, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.66@KRBTEST.COM for kudu/127.8.198.124@KRBTEST.COM
15:05:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:56 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780756, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.66@KRBTEST.COM for kudu/127.8.198.126@KRBTEST.COM
15:05:56.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.479549 9601 heartbeater.cc:346] Connected to a master server at 127.8.198.124:34423
15:05:56.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.479982 9601 heartbeater.cc:463] Registering TS with master...
15:05:56.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.481127 9601 heartbeater.cc:510] Master 127.8.198.124:34423 requested a full tablet report, sending...
15:05:56.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.482347 9602 heartbeater.cc:346] Connected to a master server at 127.8.198.125:33167
15:05:56.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.482671 9602 heartbeater.cc:463] Registering TS with master...
15:05:56.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.483507 9602 heartbeater.cc:510] Master 127.8.198.125:33167 requested a full tablet report, sending...
15:05:56.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.483770 9211 ts_manager.cc:194] Registered new tserver with Master: 314efce02ef6446094dd961b666f16b4 (127.8.198.66:37253)
15:05:56.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.485119 9211 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.8.198.66@KRBTEST.COM'} at 127.8.198.66:51381
15:05:56.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.485970 9125 ts_manager.cc:194] Registered new tserver with Master: 314efce02ef6446094dd961b666f16b4 (127.8.198.66:37253)
15:05:56.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:05:56.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.198.67 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.67.keytab.
15:05:56.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.8.198.67 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.67.keytab.
15:05:56.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.491377 8985 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:56.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:56.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-2/wal
15:05:56.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-2/data
15:05:56.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:56.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:56.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-2/logs
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-2/data/info.pb
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.67:0
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.198.67
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.67
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.67.keytab
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.67
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:56.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {KRB5CCNAME=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:05:56.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.495179 9605 heartbeater.cc:346] Connected to a master server at 127.8.198.126:39143
15:05:56.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.495472 9605 heartbeater.cc:463] Registering TS with master...
15:05:56.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.496029 9605 heartbeater.cc:510] Master 127.8.198.126:39143 requested a full tablet report, sending...
15:05:56.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.498487 9046 ts_manager.cc:194] Registered new tserver with Master: 314efce02ef6446094dd961b666f16b4 (127.8.198.66:37253)
15:05:56.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:56 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780756, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:05:56.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.680564 9025 init.cc:295] Successfully reacquired a new kerberos TGT
15:05:56.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:56.817991 9613 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:56.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:56.818426 9613 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:56.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:56.818989 9613 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:56.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:56.851071 9613 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:56.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:56.852295 9613 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.198.67
15:05:56.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.885942 9613 tablet_server_runner.cc:78] Tablet server non-default flags:
15:05:56.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:56.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:05:56.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-2/data
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-2/wal
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.67.keytab
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.67
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.67:0
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-2/data/info.pb
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.67
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.198.67
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-2/logs
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:05:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:56.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:56.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:56.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:05:56.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:05:56.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:56.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.887511 9613 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:56.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.888635 9613 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:56.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:56.906226 9619 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:57.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:57.489097 9601 heartbeater.cc:502] Master 127.8.198.124:34423 was elected leader, sending a full tablet report...
15:05:58.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:56.906924 9622 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:58.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:56.908313 9613 server_base.cc:1034] running on GCE node
15:05:58.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:56.908303 9620 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:58.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.113538 9613 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:58.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.116245 9613 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:58.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.117682 9613 hybrid_clock.cc:648] HybridClock initialized: now 1736780758117642 us; error 42 us; skew 500 ppm
15:05:58.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.118695 9613 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:58.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:58 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780758, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:05:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.125907 9613 init.cc:377] Logged in from keytab as kudu/127.8.198.67@KRBTEST.COM (short username kudu)
15:05:58.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.131497 9613 webserver.cc:458] Webserver started at http://127.8.198.67:44173/ using document root <none> and password file <none>
15:05:58.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.132732 9613 fs_manager.cc:362] Metadata directory not provided
15:05:58.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.133001 9613 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:58.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.133496 9613 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:58.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.140679 9613 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-2/data/instance:
15:05:58.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a6a098cf6c944ee09b741bae26efb0aa"
15:05:58.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:58 on dist-test-slave-s7w9"
15:05:58.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.142143 9613 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-2/wal/instance:
15:05:58.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a6a098cf6c944ee09b741bae26efb0aa"
15:05:58.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:58 on dist-test-slave-s7w9"
15:05:58.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.150542 9613 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.000s
15:05:58.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.156348 9633 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:58.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.157338 9613 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
15:05:58.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.157603 9613 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-2/data,/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-2/wal
15:05:58.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a6a098cf6c944ee09b741bae26efb0aa"
15:05:58.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:58 on dist-test-slave-s7w9"
15:05:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.157877 9613 fs_report.cc:389] FS layout report
15:05:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-2/wal
15:05:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-2/wal
15:05:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-2/data/data
15:05:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:58.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:58.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.225435 9613 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:58.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.230475 9613 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:58.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.230885 9613 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:58.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.233497 9613 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:05:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.237118 9613 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:05:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.237305 9613 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:05:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.237494 9613 ts_tablet_manager.cc:610] Registered 0 tablets
15:05:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.237675 9613 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:05:58.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:58 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780758, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:05:58.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.407614 9104 init.cc:295] Successfully reacquired a new kerberos TGT
15:05:58.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.510044 9613 rpc_server.cc:307] RPC server started. Bound to: 127.8.198.67:45947
15:05:58.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.510151 9745 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.198.67:45947 every 8 connection(s)
15:05:58.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.512902 9613 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-2/data/info.pb
15:05:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.521109 8985 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu as pid 9613
15:05:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.521591 8985 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/ts-2/wal/instance
15:05:58.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:58 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780758, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.67@KRBTEST.COM for kudu/127.8.198.124@KRBTEST.COM
15:05:58.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:58 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780758, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.67@KRBTEST.COM for kudu/127.8.198.125@KRBTEST.COM
15:05:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.571964 9746 heartbeater.cc:346] Connected to a master server at 127.8.198.124:34423
15:05:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.572459 9746 heartbeater.cc:463] Registering TS with master...
15:05:58.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.573730 9746 heartbeater.cc:510] Master 127.8.198.124:34423 requested a full tablet report, sending...
15:05:58.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.576709 9211 ts_manager.cc:194] Registered new tserver with Master: a6a098cf6c944ee09b741bae26efb0aa (127.8.198.67:45947)
15:05:58.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.578658 9211 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.8.198.67@KRBTEST.COM'} at 127.8.198.67:58705
15:05:58.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:58 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780758, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.67@KRBTEST.COM for kudu/127.8.198.126@KRBTEST.COM
15:05:58.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.595679 9747 heartbeater.cc:346] Connected to a master server at 127.8.198.125:33167
15:05:58.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.596041 9747 heartbeater.cc:463] Registering TS with master...
15:05:58.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.596721 9747 heartbeater.cc:510] Master 127.8.198.125:33167 requested a full tablet report, sending...
15:05:58.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.599294 9125 ts_manager.cc:194] Registered new tserver with Master: a6a098cf6c944ee09b741bae26efb0aa (127.8.198.67:45947)
15:05:58.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.604472 9748 heartbeater.cc:346] Connected to a master server at 127.8.198.126:39143
15:05:58.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.604781 9748 heartbeater.cc:463] Registering TS with master...
15:05:58.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.605301 9748 heartbeater.cc:510] Master 127.8.198.126:39143 requested a full tablet report, sending...
15:05:58.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.607460 9046 ts_manager.cc:194] Registered new tserver with Master: a6a098cf6c944ee09b741bae26efb0aa (127.8.198.67:45947)
15:05:58.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.612284 8985 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
15:05:58.623 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:05:58.625 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
15:05:58.633 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
15:05:58.634 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
15:05:58.636 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.8.198.126"
port: 39143
}
bound_http_address {
host: "127.8.198.126"
port: 44225
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.8.198.125"
port: 33167
}
bound_http_address {
host: "127.8.198.125"
port: 37437
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.8.198.124"
port: 34423
}
bound_http_address {
host: "127.8.198.124"
port: 36391
}
}
}
15:05:58.636 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:05:58.637 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.8.198.65"
port: 42545
}
bound_http_address {
host: "127.8.198.65"
port: 41201
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.8.198.66"
port: 37253
}
bound_http_address {
host: "127.8.198.66"
port: 39599
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.8.198.67"
port: 45947
}
bound_http_address {
host: "127.8.198.67"
port: 44173
}
}
}
15:05:58.638 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:05:58.639 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:05:58.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:58 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780758, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:05:58.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:05:58.668 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:05:58.671 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5cc
15:05:58.677 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:05:58.678 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5cc
15:05:58.679 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:05:58.682 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
15:05:58.683 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:05:58.683 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:05:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:05:58 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780758, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:05:58.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:05:58.712 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:05:58.712 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5cc
15:05:58.713 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:05:58.713 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.198.126:39143
15:05:58.714 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
15:05:58.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.715354 8985 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu with pid 9011
15:05:58.747 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:05:58.747 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.198.124:34423
15:05:58.747 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
15:05:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.747855 8985 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu with pid 9166
15:05:58.777 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:05:58.777 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.198.125:33167
15:05:58.777 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
15:05:58.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.777904 8985 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu with pid 9087
15:05:58.801 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:05:58.802 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.198.126:39143
15:05:58.806 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
15:05:58.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:58.807426 8985 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:58.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:05:58.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/wal
15:05:58.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data
15:05:58.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/logs
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data/info.pb
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.126:39143
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.126
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44225
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.126.keytab
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.126
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:05:58.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:05:58.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:05:58.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:05:58.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:58.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:58.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:58.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:58.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:05:58.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:58.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:05:58.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:05:59.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:59.136866 9760 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:59.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:59.137418 9760 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:59.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:59.138044 9760 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:59.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:59.169544 9760 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:05:59.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:59.169759 9760 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:05:59.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:59.170080 9760 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:59.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:59.170291 9760 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:05:59.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:59.170480 9760 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:05:59.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:59.170660 9760 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:05:59.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:59.170805 9760 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41287/jwks.json
15:05:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:59.207228 9760 master_runner.cc:386] Master server non-default flags:
15:05:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:05:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data
15:05:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/wal
15:05:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:05:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.126.keytab
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.126
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.126:39143
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data/info.pb
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.126
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44225
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/logs
15:05:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:05:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:05:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:05:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:59.208633 9760 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:59.209563 9760 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:59.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:59.225082 9767 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:59.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:59.501343 9602 heartbeater.cc:643] Failed to heartbeat to 127.8.198.125:33167 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111)
15:05:59.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:59.533447 9456 heartbeater.cc:643] Failed to heartbeat to 127.8.198.124:34423 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.8.198.124:34423: connect: Connection refused (error 111)
15:05:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:59.583971 9746 heartbeater.cc:502] Master 127.8.198.124:34423 was elected leader, sending a full tablet report...
15:05:59.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:59.588240 9746 heartbeater.cc:643] Failed to heartbeat to 127.8.198.124:34423 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.8.198.124:34423: connect: Connection refused (error 111)
15:06:00.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:00.100399 9597 debug-util.cc:398] Leaking SignalData structure 0x7b08000bf5c0 after lost signal to thread 9469
15:06:00.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:00.101208 9597 debug-util.cc:398] Leaking SignalData structure 0x7b08000b2d00 after lost signal to thread 9600
15:06:00.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:59.225677 9766 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:00.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:05:59.226449 9769 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:00.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:05:59.227351 9760 server_base.cc:1034] running on GCE node
15:06:00.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.474495 9760 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:00.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.480679 9760 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:00.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.482110 9760 hybrid_clock.cc:648] HybridClock initialized: now 1736780760481979 us; error 130 us; skew 500 ppm
15:06:00.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.483147 9760 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:00 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780760, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:00.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.488969 9760 init.cc:377] Logged in from keytab as kudu/127.8.198.126@KRBTEST.COM (short username kudu)
15:06:00.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.494963 9760 webserver.cc:458] Webserver started at http://127.8.198.126:44225/ using document root <none> and password file <none>
15:06:00.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.496071 9760 fs_manager.cc:362] Metadata directory not provided
15:06:00.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.496317 9760 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:00.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.534842 9760 fs_manager.cc:714] Time spent opening directory manager: real 0.036s user 0.009s sys 0.012s
15:06:00.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.549929 9788 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:00.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.551517 9760 fs_manager.cc:730] Time spent opening block manager: real 0.007s user 0.003s sys 0.003s
15:06:00.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.551956 9760 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data,/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/wal
15:06:00.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "251861861371402797d330f9ecdd6143"
15:06:00.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:46 on dist-test-slave-s7w9"
15:06:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.555449 9760 fs_report.cc:389] FS layout report
15:06:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/wal
15:06:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/wal
15:06:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data/data
15:06:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:00.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.624401 9760 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:00.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.630995 9760 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:00.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.631423 9760 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:00.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.717608 9760 rpc_server.cc:307] RPC server started. Bound to: 127.8.198.126:39143
15:06:00.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.717675 9840 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.198.126:39143 every 8 connection(s)
15:06:00.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.722111 9760 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data/info.pb
15:06:00.730 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:00.730 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.198.124:34423
15:06:00.730 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
15:06:00.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.729269 8985 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu as pid 9760
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.731719 8985 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/wal
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/logs
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data/info.pb
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:00.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.124:34423
15:06:00.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.124
15:06:00.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36391
15:06:00.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.124.keytab
15:06:00.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.124
15:06:00.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:00.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:06:00.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:06:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:06:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:06:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:06:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:06:00.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.738446 9841 sys_catalog.cc:263] Verifying existing consensus state
15:06:00.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.747498 9841 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: Bootstrap starting.
15:06:00.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.800932 9841 log.cc:826] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: Log is configured to *not* fsync() on all Append() calls
15:06:00.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.820304 9841 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:06:00.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.821316 9841 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: Bootstrap complete.
15:06:00.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.862632 9841 raft_consensus.cc:357] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:00.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.866250 9841 raft_consensus.cc:738] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 251861861371402797d330f9ecdd6143, State: Initialized, Role: FOLLOWER
15:06:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.867374 9841 consensus_queue.cc:260] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.870388 9845 sys_catalog.cc:455] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:00.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.871449 9845 sys_catalog.cc:458] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: This master's current role is: FOLLOWER
15:06:00.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.872296 9841 sys_catalog.cc:564] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: configured and running, proceeding with master startup.
15:06:00.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:00 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780760, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.126@KRBTEST.COM for kudu/127.8.198.126@KRBTEST.COM
15:06:00.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.932031 9856 catalog_manager.cc:1260] Loaded cluster ID: ef52103ef7db46f78a1ee3713871c7d3
15:06:00.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.932341 9856 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: loading cluster ID for follower catalog manager: success
15:06:00.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.948530 9856 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: acquiring CA information for follower catalog manager: success
15:06:00.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.972777 9856 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:06:00.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.954418 9760 master_runner.cc:186] Error getting master registration for 127.8.198.125:33167: Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111),
15:06:00.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:00.985206 9760 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:06:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:01.102447 9843 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:01.103085 9843 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:01.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:01.103849 9843 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:01.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:01.155441 9843 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:06:01.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:01.155681 9843 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:06:01.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:01.156061 9843 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:01.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:01.156383 9843 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:06:01.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:01.156656 9843 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:06:01.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:01.156916 9843 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:06:01.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:01.157140 9843 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41287/jwks.json
15:06:01.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:01.194854 9843 master_runner.cc:386] Master server non-default flags:
15:06:01.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:01.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:06:01.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:01.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:01.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data
15:06:01.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/wal
15:06:01.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:06:01.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:01.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:06:01.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.124.keytab
15:06:01.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.124
15:06:01.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:01.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:01.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:01.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.124:34423
15:06:01.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:01.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:01.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data/info.pb
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.124
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36391
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/logs
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:01.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:01.196406 9843 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:01.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:01.197640 9843 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:01.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:01.234987 9865 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:01.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:01.474905 8970 debug-util.cc:398] Leaking SignalData structure 0x7b08000bdd80 after lost signal to thread 8845
15:06:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:01.476317 8970 debug-util.cc:398] Leaking SignalData structure 0x7b08000bdf20 after lost signal to thread 8973
15:06:01.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:01.583117 9605 heartbeater.cc:346] Connected to a master server at 127.8.198.126:39143
15:06:01.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:01.589984 9806 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "314efce02ef6446094dd961b666f16b4" instance_seqno: 1736780756363637) as {username='kudu', principal='kudu/127.8.198.66@KRBTEST.COM'} at 127.8.198.66:47583; Asking this server to re-register.
15:06:01.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:01.592155 9605 heartbeater.cc:463] Registering TS with master...
15:06:01.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:01.601797 9806 ts_manager.cc:194] Registered new tserver with Master: 314efce02ef6446094dd961b666f16b4 (127.8.198.66:37253)
15:06:01.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:01.643349 9748 heartbeater.cc:346] Connected to a master server at 127.8.198.126:39143
15:06:01.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:01.646435 9806 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "a6a098cf6c944ee09b741bae26efb0aa" instance_seqno: 1736780758465558) as {username='kudu', principal='kudu/127.8.198.67@KRBTEST.COM'} at 127.8.198.67:47503; Asking this server to re-register.
15:06:01.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:01.648543 9748 heartbeater.cc:463] Registering TS with master...
15:06:01.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:01.652031 9806 ts_manager.cc:194] Registered new tserver with Master: a6a098cf6c944ee09b741bae26efb0aa (127.8.198.67:45947)
15:06:01.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:01.663308 9458 heartbeater.cc:346] Connected to a master server at 127.8.198.126:39143
15:06:01.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:01.666854 9806 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "4af75b1b5ad4450385a38dc956ee3969" instance_seqno: 1736780754350460) as {username='kudu', principal='kudu/127.8.198.65@KRBTEST.COM'} at 127.8.198.65:35207; Asking this server to re-register.
15:06:01.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:01.668243 9458 heartbeater.cc:463] Registering TS with master...
15:06:01.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:01.671052 9806 ts_manager.cc:194] Registered new tserver with Master: 4af75b1b5ad4450385a38dc956ee3969 (127.8.198.65:42545)
15:06:02.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:02.703305 9887 raft_consensus.cc:491] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:06:02.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:02.706622 9887 raft_consensus.cc:513] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:02.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:02.750703 9792 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111)
15:06:02.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:02.755295 9887 leader_election.cc:290] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 712570e4ca7d40e9b588643a35fd6197 (127.8.198.125:33167), a32bb1830f4f4f458a1ad99a5ed3122f (127.8.198.124:34423)
15:06:02.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:02.777923 9790 leader_election.cc:336] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer a32bb1830f4f4f458a1ad99a5ed3122f (127.8.198.124:34423): Network error: Client connection negotiation failed: client connection to 127.8.198.124:34423: connect: Connection refused (error 111)
15:06:02.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:02.794091 9792 leader_election.cc:336] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 712570e4ca7d40e9b588643a35fd6197 (127.8.198.125:33167): Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111)
15:06:02.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:02.797786 9792 leader_election.cc:304] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 251861861371402797d330f9ecdd6143; no voters: 712570e4ca7d40e9b588643a35fd6197, a32bb1830f4f4f458a1ad99a5ed3122f
15:06:02.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:02.800071 9887 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
15:06:03.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:02.628058 9864 debug-util.cc:398] Leaking SignalData structure 0x7b080001cac0 after lost signal to thread 9843
15:06:03.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:03.154587 9843 thread.cc:635] OpenStack (cloud detector) Time spent creating pthread: real 1.918s user 0.452s sys 0.618s
15:06:03.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:03.156383 9843 thread.cc:601] OpenStack (cloud detector) Time spent starting thread: real 1.920s user 0.452s sys 0.618s
15:06:03.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:01.242425 9866 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:03.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.156678 9843 server_base.cc:1034] running on GCE node
15:06:03.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:03.157414 9868 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:03.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.159382 9843 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:03.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.174934 9843 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:03.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.176354 9843 hybrid_clock.cc:648] HybridClock initialized: now 1736780763176320 us; error 52 us; skew 500 ppm
15:06:03.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.177341 9843 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:03.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:03 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780763, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:03.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.189587 9843 init.cc:377] Logged in from keytab as kudu/127.8.198.124@KRBTEST.COM (short username kudu)
15:06:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.195941 9843 webserver.cc:458] Webserver started at http://127.8.198.124:36391/ using document root <none> and password file <none>
15:06:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.196857 9843 fs_manager.cc:362] Metadata directory not provided
15:06:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.197118 9843 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:03.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.205578 9843 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.004s sys 0.000s
15:06:03.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.210125 9895 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:03.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.211078 9843 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
15:06:03.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.211364 9843 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data,/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/wal
15:06:03.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a32bb1830f4f4f458a1ad99a5ed3122f"
15:06:03.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:50 on dist-test-slave-s7w9"
15:06:03.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.213730 9843 fs_report.cc:389] FS layout report
15:06:03.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:03.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/wal
15:06:03.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/wal
15:06:03.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data/data
15:06:03.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:03.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:03.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:03.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:03.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:03.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:03.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:03.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:03.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:03.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:03.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:03.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.273262 9843 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:03.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.278980 9843 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:03.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.279444 9843 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:03.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.420240 9843 rpc_server.cc:307] RPC server started. Bound to: 127.8.198.124:34423
15:06:03.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.420413 9946 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.198.124:34423 every 8 connection(s)
15:06:03.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.423455 9843 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data/info.pb
15:06:03.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.429008 8985 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu as pid 9843
15:06:03.430 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:03.430 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.198.125:33167
15:06:03.431 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
15:06:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.433207 8985 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:06:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:06:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/wal
15:06:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data
15:06:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:03.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:03.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:03.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:03.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:03.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/logs
15:06:03.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data/info.pb
15:06:03.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:03.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:03.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:03.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:03.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:03.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:03.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:03.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:03.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:06:03.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:03.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:03.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:03.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.125:33167
15:06:03.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.125
15:06:03.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37437
15:06:03.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.125.keytab
15:06:03.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.125
15:06:03.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:03.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:06:03.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:06:03.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:06:03.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:03.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:03.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:03.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:03.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:06:03.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:03.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:06:03.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:06:03.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.460259 9947 sys_catalog.cc:263] Verifying existing consensus state
15:06:03.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.470762 9947 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: Bootstrap starting.
15:06:03.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.525753 9947 log.cc:826] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: Log is configured to *not* fsync() on all Append() calls
15:06:03.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.556538 9947 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:06:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.559443 9947 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: Bootstrap complete.
15:06:03.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.617470 9947 raft_consensus.cc:357] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:03.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.622113 9947 raft_consensus.cc:738] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: a32bb1830f4f4f458a1ad99a5ed3122f, State: Initialized, Role: FOLLOWER
15:06:03.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.623580 9947 consensus_queue.cc:260] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:03.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.633908 9955 sys_catalog.cc:455] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:03.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.638141 9955 sys_catalog.cc:458] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: This master's current role is: FOLLOWER
15:06:03.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.637719 9947 sys_catalog.cc:564] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: configured and running, proceeding with master startup.
15:06:03.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.735867 9456 heartbeater.cc:346] Connected to a master server at 127.8.198.124:34423
15:06:03.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.748983 9601 heartbeater.cc:346] Connected to a master server at 127.8.198.124:34423
15:06:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.773100 9746 heartbeater.cc:346] Connected to a master server at 127.8.198.124:34423
15:06:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.773659 9746 heartbeater.cc:502] Master 127.8.198.124:34423 was elected leader, sending a full tablet report...
15:06:03.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.828841 9971 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:06:03.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:03 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780763, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.124@KRBTEST.COM for kudu/127.8.198.126@KRBTEST.COM
15:06:03.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.894894 9843 master_runner.cc:186] Error getting master registration for 127.8.198.125:33167: Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111),
15:06:03.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:03.922082 9843 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:06:04.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:04 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780764, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:04.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.103912 9339 init.cc:295] Successfully reacquired a new kerberos TGT
15:06:04.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:04.114573 9949 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:04.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:04.115307 9949 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:04.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:04.116276 9949 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:04.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:04.164557 9949 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:06:04.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:04.164901 9949 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:06:04.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:04.165362 9949 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:04.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:04.165680 9949 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:06:04.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:04.166025 9949 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:06:04.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:04.166311 9949 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:06:04.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:04.166522 9949 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41287/jwks.json
15:06:04.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.205535 9949 master_runner.cc:386] Master server non-default flags:
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/wal
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.125.keytab
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.125
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.125:33167
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data/info.pb
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.125
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37437
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:04.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:04.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:04.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:04.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/logs
15:06:04.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:04.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:04.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:04.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:06:04.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:06:04.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:06:04.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:04.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:06:04.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:06:04.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:04.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.207201 9949 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:04.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.208326 9949 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:04.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:04.226660 9980 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.672376 9988 raft_consensus.cc:491] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:06:04.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.673393 9988 raft_consensus.cc:513] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:04.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.696913 9988 leader_election.cc:290] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 712570e4ca7d40e9b588643a35fd6197 (127.8.198.125:33167), a32bb1830f4f4f458a1ad99a5ed3122f (127.8.198.124:34423)
15:06:04.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.748415 9912 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "4af75b1b5ad4450385a38dc956ee3969" instance_seqno: 1736780754350460) as {username='kudu', principal='kudu/127.8.198.65@KRBTEST.COM'} at 127.8.198.65:48057; Asking this server to re-register.
15:06:04.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.750587 9456 heartbeater.cc:463] Registering TS with master...
15:06:04.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.755673 9911 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "314efce02ef6446094dd961b666f16b4" instance_seqno: 1736780756363637) as {username='kudu', principal='kudu/127.8.198.66@KRBTEST.COM'} at 127.8.198.66:44863; Asking this server to re-register.
15:06:04.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.757336 9601 heartbeater.cc:463] Registering TS with master...
15:06:04.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.760524 9911 ts_manager.cc:194] Registered new tserver with Master: 314efce02ef6446094dd961b666f16b4 (127.8.198.66:37253)
15:06:04.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.758742 9912 ts_manager.cc:194] Registered new tserver with Master: 4af75b1b5ad4450385a38dc956ee3969 (127.8.198.65:42545)
15:06:04.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.777006 9746 heartbeater.cc:502] Master 127.8.198.124:34423 was elected leader, sending a full tablet report...
15:06:04.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.779498 9912 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "a6a098cf6c944ee09b741bae26efb0aa" instance_seqno: 1736780758465558) as {username='kudu', principal='kudu/127.8.198.67@KRBTEST.COM'} at 127.8.198.67:52129; Asking this server to re-register.
15:06:04.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.780829 9746 heartbeater.cc:463] Registering TS with master...
15:06:04.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.785549 9911 ts_manager.cc:194] Registered new tserver with Master: a6a098cf6c944ee09b741bae26efb0aa (127.8.198.67:45947)
15:06:04.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:04 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780760, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.126@KRBTEST.COM for kudu/127.8.198.124@KRBTEST.COM
15:06:04.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:04.808341 9792 leader_election.cc:336] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 712570e4ca7d40e9b588643a35fd6197 (127.8.198.125:33167): Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111)
15:06:04.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.814491 9922 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "251861861371402797d330f9ecdd6143" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" is_pre_election: true
15:06:04.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.815359 9922 raft_consensus.cc:2463] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 251861861371402797d330f9ecdd6143 in term 1.
15:06:04.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.816686 9790 leader_election.cc:304] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 251861861371402797d330f9ecdd6143, a32bb1830f4f4f458a1ad99a5ed3122f; no voters: 712570e4ca7d40e9b588643a35fd6197
15:06:04.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.817948 9988 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 1 FOLLOWER]: Leader pre-election won for term 2
15:06:04.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.818280 9988 raft_consensus.cc:491] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:06:04.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.818594 9988 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 1 FOLLOWER]: Advancing to term 2
15:06:04.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.827239 9988 raft_consensus.cc:513] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:04.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.832990 9922 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "251861861371402797d330f9ecdd6143" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f"
15:06:04.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.833479 9922 raft_consensus.cc:3054] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 1 FOLLOWER]: Advancing to term 2
15:06:04.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.843015 9971 catalog_manager.cc:1260] Loaded cluster ID: ef52103ef7db46f78a1ee3713871c7d3
15:06:04.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.843238 9971 catalog_manager.cc:1553] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: loading cluster ID for follower catalog manager: success
15:06:04.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.849445 9922 raft_consensus.cc:2463] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 251861861371402797d330f9ecdd6143 in term 2.
15:06:04.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:04.859042 9792 leader_election.cc:336] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 712570e4ca7d40e9b588643a35fd6197 (127.8.198.125:33167): Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111)
15:06:04.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.859102 9988 leader_election.cc:290] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 2 election: Requested vote from peers 712570e4ca7d40e9b588643a35fd6197 (127.8.198.125:33167), a32bb1830f4f4f458a1ad99a5ed3122f (127.8.198.124:34423)
15:06:04.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.862735 9790 leader_election.cc:304] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 251861861371402797d330f9ecdd6143, a32bb1830f4f4f458a1ad99a5ed3122f; no voters: 712570e4ca7d40e9b588643a35fd6197
15:06:04.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.863629 9988 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 2 FOLLOWER]: Leader election won for term 2
15:06:04.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.869238 9971 catalog_manager.cc:1575] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: acquiring CA information for follower catalog manager: success
15:06:04.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.874065 9988 raft_consensus.cc:695] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 2 LEADER]: Becoming Leader. State: Replica: 251861861371402797d330f9ecdd6143, State: Running, Role: LEADER
15:06:04.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.875841 9988 consensus_queue.cc:237] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:04.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.896263 9971 catalog_manager.cc:1603] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:06:04.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.911674 9996 sys_catalog.cc:455] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 251861861371402797d330f9ecdd6143. Latest consensus state: current_term: 2 leader_uuid: "251861861371402797d330f9ecdd6143" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:04.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.912719 9996 sys_catalog.cc:458] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: This master's current role is: LEADER
15:06:04.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.930733 9998 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
15:06:04.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.942917 9998 catalog_manager.cc:1485] Initializing Kudu cluster ID...
15:06:04.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.949803 9998 catalog_manager.cc:1260] Loaded cluster ID: ef52103ef7db46f78a1ee3713871c7d3
15:06:04.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.950079 9998 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
15:06:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.955888 9998 catalog_manager.cc:1505] Loading token signing keys...
15:06:04.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.961247 9998 catalog_manager.cc:5910] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: Loaded TSK: 0
15:06:04.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:04.962873 9998 catalog_manager.cc:1515] Initializing in-progress tserver states...
15:06:05.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:05.277006 9792 consensus_peers.cc:487] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 -> Peer 712570e4ca7d40e9b588643a35fd6197 (127.8.198.125:33167): Couldn't send request to peer 712570e4ca7d40e9b588643a35fd6197. Status: Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
15:06:05.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:05.280241 9922 raft_consensus.cc:1270] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 2 FOLLOWER]: Refusing update from remote peer 251861861371402797d330f9ecdd6143: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
15:06:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:05.282616 9996 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
15:06:05.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:05.343521 9988 sys_catalog.cc:455] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "251861861371402797d330f9ecdd6143" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:05.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:05.344465 9988 sys_catalog.cc:458] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: This master's current role is: LEADER
15:06:05.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:05.355111 9999 sys_catalog.cc:455] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 251861861371402797d330f9ecdd6143. Latest consensus state: current_term: 2 leader_uuid: "251861861371402797d330f9ecdd6143" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:05.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:05.363981 9999 sys_catalog.cc:458] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: This master's current role is: FOLLOWER
15:06:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:05.362814 10003 sys_catalog.cc:455] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "251861861371402797d330f9ecdd6143" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:05.366312 10003 sys_catalog.cc:458] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: This master's current role is: FOLLOWER
15:06:05.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:05.993273 9760 master_runner.cc:186] Error getting master registration for 127.8.198.125:33167: Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111),
15:06:05.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:05.993739 9760 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:06:06.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:06 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780766, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:06.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.140282 9484 init.cc:295] Successfully reacquired a new kerberos TGT
15:06:06.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:05.621502 9979 debug-util.cc:398] Leaking SignalData structure 0x7b080001cac0 after lost signal to thread 9949
15:06:06.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:06.267069 9949 thread.cc:635] OpenStack (cloud detector) Time spent creating pthread: real 2.037s user 0.410s sys 0.694s
15:06:06.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:06.268728 9949 thread.cc:601] OpenStack (cloud detector) Time spent starting thread: real 2.039s user 0.410s sys 0.694s
15:06:06.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:06.269032 9982 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2033 milliseconds
15:06:06.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:04.230290 9981 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:06.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:06.269975 9983 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:06.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.269927 9949 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:06:06.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.274164 9949 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:06.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.276985 9949 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:06.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.278395 9949 hybrid_clock.cc:648] HybridClock initialized: now 1736780766278310 us; error 87 us; skew 500 ppm
15:06:06.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.279227 9949 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:06.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:06 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780766, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:06.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.285043 9949 init.cc:377] Logged in from keytab as kudu/127.8.198.125@KRBTEST.COM (short username kudu)
15:06:06.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.290477 9949 webserver.cc:458] Webserver started at http://127.8.198.125:37437/ using document root <none> and password file <none>
15:06:06.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.291465 9949 fs_manager.cc:362] Metadata directory not provided
15:06:06.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.291707 9949 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:06.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.300000 9949 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
15:06:06.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.304147 10013 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:06.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.305163 9949 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.002s
15:06:06.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.305428 9949 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data,/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/wal
15:06:06.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "712570e4ca7d40e9b588643a35fd6197"
15:06:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:48 on dist-test-slave-s7w9"
15:06:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.307735 9949 fs_report.cc:389] FS layout report
15:06:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/wal
15:06:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/wal
15:06:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data/data
15:06:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:06.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:06.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:06.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:06.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.374447 9949 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:06.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.379498 9949 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.379876 9949 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:06.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.477062 9949 rpc_server.cc:307] RPC server started. Bound to: 127.8.198.125:33167
15:06:06.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.477514 10064 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.198.125:33167 every 8 connection(s)
15:06:06.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.479830 9949 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data/info.pb
15:06:06.487 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:06.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.486667 8985 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu as pid 9949
15:06:06.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.506283 10065 sys_catalog.cc:263] Verifying existing consensus state
15:06:06.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.516068 10065 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: Bootstrap starting.
15:06:06.579 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.198.126:39143(127.8.198.126:39143)] connecting to peer
15:06:06.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.585263 10065 log.cc:826] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: Log is configured to *not* fsync() on all Append() calls
15:06:06.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:06 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780760, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.126@KRBTEST.COM for kudu/127.8.198.125@KRBTEST.COM
15:06:06.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.612354 10065 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:06:06.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.613312 10065 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: Bootstrap complete.
15:06:06.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.637959 9605 heartbeater.cc:502] Master 127.8.198.126:39143 was elected leader, sending a full tablet report...
15:06:06.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.643590 10065 raft_consensus.cc:357] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:06.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.647111 10065 raft_consensus.cc:738] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 712570e4ca7d40e9b588643a35fd6197, State: Initialized, Role: FOLLOWER
15:06:06.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.648020 10065 consensus_queue.cc:260] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:06.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.673662 10065 sys_catalog.cc:564] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: configured and running, proceeding with master startup.
15:06:06.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.674371 10040 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [term 1 FOLLOWER]: Advancing to term 2
15:06:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:06 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780766, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.66@KRBTEST.COM for kudu/127.8.198.125@KRBTEST.COM
15:06:06.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.715910 9602 heartbeater.cc:346] Connected to a master server at 127.8.198.125:33167
15:06:06.732 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.198.124:34423(127.8.198.124:34423)] connecting to peer
15:06:06.733 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.198.125:33167(127.8.198.125:33167)] connecting to peer
15:06:06.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:06 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780764, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.65@KRBTEST.COM for kudu/127.8.198.125@KRBTEST.COM
15:06:06.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.756448 10082 sys_catalog.cc:455] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "251861861371402797d330f9ecdd6143" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:06.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.757973 10082 sys_catalog.cc:458] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: This master's current role is: FOLLOWER
15:06:06.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.759433 10075 sys_catalog.cc:455] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 251861861371402797d330f9ecdd6143. Latest consensus state: current_term: 2 leader_uuid: "251861861371402797d330f9ecdd6143" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:06.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.768435 10075 sys_catalog.cc:458] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: This master's current role is: FOLLOWER
15:06:06.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.766772 10071 sys_catalog.cc:455] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 leader_uuid: "251861861371402797d330f9ecdd6143" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:06.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.771571 10071 sys_catalog.cc:458] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: This master's current role is: FOLLOWER
15:06:06.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.773319 9458 heartbeater.cc:502] Master 127.8.198.126:39143 was elected leader, sending a full tablet report...
15:06:06.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.774438 9748 heartbeater.cc:502] Master 127.8.198.126:39143 was elected leader, sending a full tablet report...
15:06:06.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.781141 9457 heartbeater.cc:346] Connected to a master server at 127.8.198.125:33167
15:06:06.822 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.8.198.126:39143(127.8.198.126:39143)] Successfully connected to peer
15:06:06.825 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.8.198.124:34423(127.8.198.124:34423)] Successfully connected to peer
15:06:06.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.829625 10094 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:06:06.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:06 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780766, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.125@KRBTEST.COM for kudu/127.8.198.126@KRBTEST.COM
15:06:06.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.902215 9747 heartbeater.cc:346] Connected to a master server at 127.8.198.125:33167
15:06:06.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.910856 10030 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "a6a098cf6c944ee09b741bae26efb0aa" instance_seqno: 1736780758465558) as {username='kudu', principal='kudu/127.8.198.67@KRBTEST.COM'} at 127.8.198.67:47041; Asking this server to re-register.
15:06:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.913095 9747 heartbeater.cc:463] Registering TS with master...
15:06:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:06 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780766, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.125@KRBTEST.COM for kudu/127.8.198.125@KRBTEST.COM
15:06:06.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:06.916431 10030 ts_manager.cc:194] Registered new tserver with Master: a6a098cf6c944ee09b741bae26efb0aa (127.8.198.67:45947)
15:06:06.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:06 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780766, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.125@KRBTEST.COM for kudu/127.8.198.124@KRBTEST.COM
15:06:06.966 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.8.198.125:33167(127.8.198.125:33167)] Successfully connected to peer
15:06:06.986 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
15:06:06.998 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.8.198.126
15:06:06.999 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.8.198.124
15:06:07.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:07.040151 9996 sys_catalog.cc:455] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "251861861371402797d330f9ecdd6143" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:07.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:07.042788 9996 sys_catalog.cc:458] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: This master's current role is: LEADER
15:06:07.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:07 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780758, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.198.124@KRBTEST.COM
15:06:07.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:07 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780758, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.198.126@KRBTEST.COM
15:06:07.283 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.8.198.125
15:06:07.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:07 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780758, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.198.125@KRBTEST.COM
15:06:07.335 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xbacfa658, L:/127.0.0.1:46262 - R:/127.8.198.124:34423] using SASL/GSSAPI
15:06:07.342 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x91a01cb4, L:/127.0.0.1:57634 - R:/127.8.198.126:39143] using SASL/GSSAPI
15:06:07.352 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x5a89e978, L:/127.0.0.1:55212 - R:/127.8.198.125:33167] using SASL/GSSAPI
15:06:07.401 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
15:06:07.402 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=910), Traces: [0ms] refreshing cache from master, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.198.126:39143, [235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.198.124:34423, [236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.8.198.125:33167, [864ms] Sub RPC ConnectToMaster: received response from server master-127.8.198.124:34423: OK, [864ms] Sub RPC ConnectToMaster: received response from server master-127.8.198.125:33167: OK, [868ms] Sub RPC ConnectToMaster: received response from server master-127.8.198.126:39143: OK, deferred=null) after lookup
15:06:07.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:07.726681 10030 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "314efce02ef6446094dd961b666f16b4" instance_seqno: 1736780756363637) as {username='kudu', principal='kudu/127.8.198.66@KRBTEST.COM'} at 127.8.198.66:56637; Asking this server to re-register.
15:06:07.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:07.727926 9602 heartbeater.cc:463] Registering TS with master...
15:06:07.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:07.731319 10030 ts_manager.cc:194] Registered new tserver with Master: 314efce02ef6446094dd961b666f16b4 (127.8.198.66:37253)
15:06:07.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:07.786818 10030 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "4af75b1b5ad4450385a38dc956ee3969" instance_seqno: 1736780754350460) as {username='kudu', principal='kudu/127.8.198.65@KRBTEST.COM'} at 127.8.198.65:53723; Asking this server to re-register.
15:06:07.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:07.787847 9457 heartbeater.cc:463] Registering TS with master...
15:06:07.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:07.790136 10030 ts_manager.cc:194] Registered new tserver with Master: 4af75b1b5ad4450385a38dc956ee3969 (127.8.198.65:42545)
15:06:07.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:07.837409 10094 catalog_manager.cc:1260] Loaded cluster ID: ef52103ef7db46f78a1ee3713871c7d3
15:06:07.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:07.837613 10094 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: loading cluster ID for follower catalog manager: success
15:06:07.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:07.842551 10094 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: acquiring CA information for follower catalog manager: success
15:06:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:07.846284 10094 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:06:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:08 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780768, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:08.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:08.130334 9629 init.cc:295] Successfully reacquired a new kerberos TGT
15:06:08.430 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:06:08.431 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:06:08.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:08 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780768, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:08.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:06:08.460 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:08.460 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5cc
15:06:08.462 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:06:08.462 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.198.126:39143
15:06:08.462 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
15:06:08.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:08.463187 8985 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu with pid 9760
15:06:08.510 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.8.198.126:39143(127.8.198.126:39143)] handling channelInactive
15:06:08.510 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.198.126:39143(127.8.198.126:39143)] cleaning up while in state READY due to: connection closed
15:06:08.514 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:08.514 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.198.124:34423
15:06:08.514 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
15:06:08.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:08.515772 8985 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu with pid 9843
15:06:08.548 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:08.548 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.198.125:33167
15:06:08.548 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.8.198.124:34423(127.8.198.124:34423)] handling channelInactive
15:06:08.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.198.124:34423(127.8.198.124:34423)] cleaning up while in state READY due to: connection closed
15:06:08.549 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
15:06:08.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:08.549933 8985 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu with pid 9949
15:06:08.570 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.8.198.125:33167(127.8.198.125:33167)] handling channelInactive
15:06:08.570 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:08.570 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.198.126:39143
15:06:08.570 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.198.125:33167(127.8.198.125:33167)] cleaning up while in state READY due to: connection closed
15:06:08.570 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
15:06:08.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:08.572127 8985 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:06:08.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:06:08.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/wal
15:06:08.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data
15:06:08.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:08.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:08.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:08.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:08.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:08.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:08.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/logs
15:06:08.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data/info.pb
15:06:08.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:08.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:08.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:08.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:08.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:08.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:06:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.126:39143
15:06:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.126
15:06:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44225
15:06:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.126.keytab
15:06:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.126
15:06:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:06:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:06:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:06:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:06:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:06:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:06:08.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:08.880781 10106 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:08.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:08.881368 10106 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:08.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:08.882182 10106 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:08.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:08.913182 10106 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:06:08.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:08.913383 10106 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:06:08.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:08.913630 10106 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:08.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:08.913815 10106 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:06:08.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:08.914060 10106 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:06:08.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:08.914364 10106 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:06:08.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:08.914547 10106 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41287/jwks.json
15:06:08.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:08.948962 10106 master_runner.cc:386] Master server non-default flags:
15:06:08.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:08.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/wal
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.126.keytab
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.126
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.126:39143
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data/info.pb
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.126
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44225
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/logs
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:08.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:06:08.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:06:08.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:06:08.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:08.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:06:08.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:06:08.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:08.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:08.950421 10106 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:08.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:08.951385 10106 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:08.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:08.967160 10119 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:08.968564 10116 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:08.969365 10106 server_base.cc:1034] running on GCE node
15:06:10.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:08.967700 10117 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.127038 10106 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:10.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.130257 10106 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:10.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.131644 10106 hybrid_clock.cc:648] HybridClock initialized: now 1736780770131599 us; error 57 us; skew 500 ppm
15:06:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.132715 10106 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:10.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:10 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780770, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:10.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.145406 10106 init.cc:377] Logged in from keytab as kudu/127.8.198.126@KRBTEST.COM (short username kudu)
15:06:10.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.151250 10106 webserver.cc:458] Webserver started at http://127.8.198.126:44225/ using document root <none> and password file <none>
15:06:10.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.152541 10106 fs_manager.cc:362] Metadata directory not provided
15:06:10.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.152837 10106 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:10.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.164289 10106 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.004s
15:06:10.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.168623 10131 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:10.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.169799 10106 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
15:06:10.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.170120 10106 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data,/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/wal
15:06:10.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "251861861371402797d330f9ecdd6143"
15:06:10.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:46 on dist-test-slave-s7w9"
15:06:10.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.172396 10106 fs_report.cc:389] FS layout report
15:06:10.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/wal
15:06:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/wal
15:06:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data/data
15:06:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:10.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.233006 10106 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.238279 10106 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:10.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.238723 10106 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:10.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.304106 10106 rpc_server.cc:307] RPC server started. Bound to: 127.8.198.126:39143
15:06:10.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.304220 10183 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.198.126:39143 every 8 connection(s)
15:06:10.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.306780 10106 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data/info.pb
15:06:10.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.311165 8985 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu as pid 10106
15:06:10.311 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:10.311 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.198.124:34423
15:06:10.312 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
15:06:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.313167 8985 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:06:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:06:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/wal
15:06:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data
15:06:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/logs
15:06:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data/info.pb
15:06:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:06:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.124:34423
15:06:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.124
15:06:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36391
15:06:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.124.keytab
15:06:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.124
15:06:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:06:10.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:06:10.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:06:10.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:10.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:10.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:10.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:10.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:06:10.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:10.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:06:10.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:06:10.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.316819 10184 sys_catalog.cc:263] Verifying existing consensus state
15:06:10.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.323916 10184 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: Bootstrap starting.
15:06:10.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.359627 10184 log.cc:826] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: Log is configured to *not* fsync() on all Append() calls
15:06:10.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.375412 10184 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:06:10.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.376060 10184 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: Bootstrap complete.
15:06:10.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.402463 10184 raft_consensus.cc:357] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:10.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.405797 10184 raft_consensus.cc:738] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 251861861371402797d330f9ecdd6143, State: Initialized, Role: FOLLOWER
15:06:10.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.406631 10184 consensus_queue.cc:260] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:10.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.409457 10188 sys_catalog.cc:455] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:10.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.410302 10188 sys_catalog.cc:458] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: This master's current role is: FOLLOWER
15:06:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.411214 10184 sys_catalog.cc:564] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: configured and running, proceeding with master startup.
15:06:10.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.442991 10199 catalog_manager.cc:1260] Loaded cluster ID: ef52103ef7db46f78a1ee3713871c7d3
15:06:10.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.447467 10199 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: loading cluster ID for follower catalog manager: success
15:06:10.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.457980 10199 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: acquiring CA information for follower catalog manager: success
15:06:10.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.464437 10199 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:06:10.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:10 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780770, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.126@KRBTEST.COM for kudu/127.8.198.126@KRBTEST.COM
15:06:10.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.482578 10106 master_runner.cc:186] Error getting master registration for 127.8.198.125:33167: Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111),
15:06:10.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.522881 10106 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:06:10.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.679260 9605 heartbeater.cc:346] Connected to a master server at 127.8.198.126:39143
15:06:10.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.683838 10149 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "314efce02ef6446094dd961b666f16b4" instance_seqno: 1736780756363637) as {username='kudu', principal='kudu/127.8.198.66@KRBTEST.COM'} at 127.8.198.66:58369; Asking this server to re-register.
15:06:10.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.685416 9605 heartbeater.cc:463] Registering TS with master...
15:06:10.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.687779 10149 ts_manager.cc:194] Registered new tserver with Master: 314efce02ef6446094dd961b666f16b4 (127.8.198.66:37253)
15:06:10.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:10.716810 10186 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:10.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:10.717532 10186 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:10.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:10.718538 10186 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:10.773226 10186 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:06:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:10.773524 10186 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:06:10.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:10.774021 10186 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:10.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:10.774365 10186 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:06:10.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:10.774680 10186 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:06:10.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:10.774996 10186 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:06:10.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:10.775238 10186 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41287/jwks.json
15:06:10.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.838256 10186 master_runner.cc:386] Master server non-default flags:
15:06:10.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:10.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:06:10.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:10.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/wal
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.124.keytab
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.124
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.124:34423
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data/info.pb
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.124
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36391
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/logs
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:10.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:10.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:10.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:06:10.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:06:10.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:06:10.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:10.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:06:10.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:06:10.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:10.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.839952 10186 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:10.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.841194 10186 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:10.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.847915 9748 heartbeater.cc:346] Connected to a master server at 127.8.198.126:39143
15:06:10.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.850545 10149 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "a6a098cf6c944ee09b741bae26efb0aa" instance_seqno: 1736780758465558) as {username='kudu', principal='kudu/127.8.198.67@KRBTEST.COM'} at 127.8.198.67:38673; Asking this server to re-register.
15:06:10.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.852267 9748 heartbeater.cc:463] Registering TS with master...
15:06:10.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.855171 10149 ts_manager.cc:194] Registered new tserver with Master: a6a098cf6c944ee09b741bae26efb0aa (127.8.198.67:45947)
15:06:10.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.865341 9458 heartbeater.cc:346] Connected to a master server at 127.8.198.126:39143
15:06:10.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.867745 10149 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "4af75b1b5ad4450385a38dc956ee3969" instance_seqno: 1736780754350460) as {username='kudu', principal='kudu/127.8.198.65@KRBTEST.COM'} at 127.8.198.65:47527; Asking this server to re-register.
15:06:10.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.870064 9458 heartbeater.cc:463] Registering TS with master...
15:06:10.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.872952 10149 ts_manager.cc:194] Registered new tserver with Master: 4af75b1b5ad4450385a38dc956ee3969 (127.8.198.65:42545)
15:06:10.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:10.876085 10215 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:11.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:11.893518 10227 raft_consensus.cc:491] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:06:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:11.894114 10227 raft_consensus.cc:513] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:11.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:11.909214 10135 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111)
15:06:11.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:11.910296 10227 leader_election.cc:290] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 712570e4ca7d40e9b588643a35fd6197 (127.8.198.125:33167), a32bb1830f4f4f458a1ad99a5ed3122f (127.8.198.124:34423)
15:06:11.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:11.919911 10135 leader_election.cc:336] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 712570e4ca7d40e9b588643a35fd6197 (127.8.198.125:33167): Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111)
15:06:11.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:11.920639 10133 leader_election.cc:336] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer a32bb1830f4f4f458a1ad99a5ed3122f (127.8.198.124:34423): Network error: Client connection negotiation failed: client connection to 127.8.198.124:34423: connect: Connection refused (error 111)
15:06:11.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:11.920984 10133 leader_election.cc:304] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 251861861371402797d330f9ecdd6143; no voters: 712570e4ca7d40e9b588643a35fd6197, a32bb1830f4f4f458a1ad99a5ed3122f
15:06:11.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:11.921861 10227 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
15:06:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:10.880285 10214 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:10.882299 10186 server_base.cc:1034] running on GCE node
15:06:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:10.881600 10218 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.187424 10186 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:12.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.190466 10186 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:12.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.191869 10186 hybrid_clock.cc:648] HybridClock initialized: now 1736780772191810 us; error 74 us; skew 500 ppm
15:06:12.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.192754 10186 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:12.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:12 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780772, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:12.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.198822 10186 init.cc:377] Logged in from keytab as kudu/127.8.198.124@KRBTEST.COM (short username kudu)
15:06:12.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.204788 10186 webserver.cc:458] Webserver started at http://127.8.198.124:36391/ using document root <none> and password file <none>
15:06:12.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.205792 10186 fs_manager.cc:362] Metadata directory not provided
15:06:12.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.206061 10186 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.218042 10186 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.001s sys 0.004s
15:06:12.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.222726 10237 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:12.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.223817 10186 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
15:06:12.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.224179 10186 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data,/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/wal
15:06:12.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a32bb1830f4f4f458a1ad99a5ed3122f"
15:06:12.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:50 on dist-test-slave-s7w9"
15:06:12.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.227300 10186 fs_report.cc:389] FS layout report
15:06:12.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:12.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/wal
15:06:12.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/wal
15:06:12.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data/data
15:06:12.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:12.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:12.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:12.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:12.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:12.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:12.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:12.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:12.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:12.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:12.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:12.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.297199 10186 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:12.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.303647 10186 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:12.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.304126 10186 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:12.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.394523 10186 rpc_server.cc:307] RPC server started. Bound to: 127.8.198.124:34423
15:06:12.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.394660 10289 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.198.124:34423 every 8 connection(s)
15:06:12.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.399060 8985 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu as pid 10186
15:06:12.399 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:12.399 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.198.125:33167
15:06:12.400 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
15:06:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.398298 10186 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data/info.pb
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.401738 8985 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/wal
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/logs
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data/info.pb
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.125:33167
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.125
15:06:12.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37437
15:06:12.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.125.keytab
15:06:12.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.125
15:06:12.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:12.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:06:12.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:06:12.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:06:12.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:12.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:12.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:12.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:12.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:06:12.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:12.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:06:12.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:06:12.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.426801 10290 sys_catalog.cc:263] Verifying existing consensus state
15:06:12.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.434787 10290 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: Bootstrap starting.
15:06:12.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.485462 10290 log.cc:826] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: Log is configured to *not* fsync() on all Append() calls
15:06:12.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.507138 10290 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:06:12.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.508219 10290 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: Bootstrap complete.
15:06:12.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.555207 10290 raft_consensus.cc:357] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:12.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.558944 10290 raft_consensus.cc:738] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: a32bb1830f4f4f458a1ad99a5ed3122f, State: Initialized, Role: FOLLOWER
15:06:12.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.559906 10290 consensus_queue.cc:260] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:12.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.586989 10294 sys_catalog.cc:455] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:12.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.587904 10294 sys_catalog.cc:458] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: This master's current role is: FOLLOWER
15:06:12.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.589404 10290 sys_catalog.cc:564] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: configured and running, proceeding with master startup.
15:06:12.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.635782 10305 catalog_manager.cc:1260] Loaded cluster ID: ef52103ef7db46f78a1ee3713871c7d3
15:06:12.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.636039 10305 catalog_manager.cc:1553] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: loading cluster ID for follower catalog manager: success
15:06:12.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.644244 10305 catalog_manager.cc:1575] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: acquiring CA information for follower catalog manager: success
15:06:12.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.650180 10305 catalog_manager.cc:1603] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:06:12.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:12 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780772, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.124@KRBTEST.COM for kudu/127.8.198.126@KRBTEST.COM
15:06:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.679625 10186 master_runner.cc:186] Error getting master registration for 127.8.198.125:33167: Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111),
15:06:12.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.708945 10186 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:06:12.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.808511 9601 heartbeater.cc:346] Connected to a master server at 127.8.198.124:34423
15:06:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:12.811086 10291 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:12.811699 10291 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:12.812625 10291 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:12.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.815220 10255 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "314efce02ef6446094dd961b666f16b4" instance_seqno: 1736780756363637) as {username='kudu', principal='kudu/127.8.198.66@KRBTEST.COM'} at 127.8.198.66:34865; Asking this server to re-register.
15:06:12.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.816571 9601 heartbeater.cc:463] Registering TS with master...
15:06:12.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.819478 10255 ts_manager.cc:194] Registered new tserver with Master: 314efce02ef6446094dd961b666f16b4 (127.8.198.66:37253)
15:06:12.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:12.859202 10291 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:06:12.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:12.859510 10291 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:06:12.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:12.859946 10291 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:12.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:12.860261 10291 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:06:12.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:12.860555 10291 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:06:12.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:12.860848 10291 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:06:12.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:12.861064 10291 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41287/jwks.json
15:06:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.910658 9746 heartbeater.cc:346] Connected to a master server at 127.8.198.124:34423
15:06:12.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.914317 10255 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "a6a098cf6c944ee09b741bae26efb0aa" instance_seqno: 1736780758465558) as {username='kudu', principal='kudu/127.8.198.67@KRBTEST.COM'} at 127.8.198.67:49553; Asking this server to re-register.
15:06:12.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.916397 9746 heartbeater.cc:463] Registering TS with master...
15:06:12.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.920007 10255 ts_manager.cc:194] Registered new tserver with Master: a6a098cf6c944ee09b741bae26efb0aa (127.8.198.67:45947)
15:06:12.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.923251 10291 master_runner.cc:386] Master server non-default flags:
15:06:12.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:12.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:06:12.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:12.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:12.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data
15:06:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/wal
15:06:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:06:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:06:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.125.keytab
15:06:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.125
15:06:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.125:33167
15:06:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:06:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:06:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:06:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data/info.pb
15:06:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.125
15:06:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37437
15:06:12.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:12.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:12.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:12.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:12.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:12.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/logs
15:06:12.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:12.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:12.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:12.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:06:12.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:06:12.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:06:12.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:12.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:06:12.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:06:12.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:12.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.924934 10291 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:12.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.926204 10291 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:12.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.949591 9456 heartbeater.cc:346] Connected to a master server at 127.8.198.124:34423
15:06:12.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.952557 10255 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "4af75b1b5ad4450385a38dc956ee3969" instance_seqno: 1736780754350460) as {username='kudu', principal='kudu/127.8.198.65@KRBTEST.COM'} at 127.8.198.65:54977; Asking this server to re-register.
15:06:12.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.954460 9456 heartbeater.cc:463] Registering TS with master...
15:06:12.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:12.957549 10255 ts_manager.cc:194] Registered new tserver with Master: 4af75b1b5ad4450385a38dc956ee3969 (127.8.198.65:42545)
15:06:12.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:12.959352 10321 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:13.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.780498 10328 raft_consensus.cc:491] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:06:13.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.781647 10328 raft_consensus.cc:513] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:13.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.811789 10328 leader_election.cc:290] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 712570e4ca7d40e9b588643a35fd6197 (127.8.198.125:33167), a32bb1830f4f4f458a1ad99a5ed3122f (127.8.198.124:34423)
15:06:13.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.837095 10265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "251861861371402797d330f9ecdd6143" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" is_pre_election: true
15:06:13.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.838356 10265 raft_consensus.cc:2463] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 251861861371402797d330f9ecdd6143 in term 2.
15:06:13.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.840163 10133 leader_election.cc:304] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 251861861371402797d330f9ecdd6143, a32bb1830f4f4f458a1ad99a5ed3122f; no voters:
15:06:13.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.841512 10328 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 2 FOLLOWER]: Leader pre-election won for term 3
15:06:13.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.842209 10328 raft_consensus.cc:491] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:06:13.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.844180 10328 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 2 FOLLOWER]: Advancing to term 3
15:06:13.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:13.861656 10135 leader_election.cc:336] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 712570e4ca7d40e9b588643a35fd6197 (127.8.198.125:33167): Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111)
15:06:13.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.863540 10328 raft_consensus.cc:513] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:13.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:13.874601 10135 leader_election.cc:336] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 712570e4ca7d40e9b588643a35fd6197 (127.8.198.125:33167): Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111)
15:06:13.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.877656 10265 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "251861861371402797d330f9ecdd6143" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f"
15:06:13.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.878506 10265 raft_consensus.cc:3054] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 2 FOLLOWER]: Advancing to term 3
15:06:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.902063 10328 leader_election.cc:290] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 3 election: Requested vote from peers 712570e4ca7d40e9b588643a35fd6197 (127.8.198.125:33167), a32bb1830f4f4f458a1ad99a5ed3122f (127.8.198.124:34423)
15:06:13.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.908880 10265 raft_consensus.cc:2463] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 251861861371402797d330f9ecdd6143 in term 3.
15:06:13.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.922801 10133 leader_election.cc:304] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 251861861371402797d330f9ecdd6143, a32bb1830f4f4f458a1ad99a5ed3122f; no voters: 712570e4ca7d40e9b588643a35fd6197
15:06:13.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.923976 10328 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 3 FOLLOWER]: Leader election won for term 3
15:06:13.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.934728 10328 raft_consensus.cc:695] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 3 LEADER]: Becoming Leader. State: Replica: 251861861371402797d330f9ecdd6143, State: Running, Role: LEADER
15:06:13.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.936311 10328 consensus_queue.cc:237] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:13.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.953995 10334 sys_catalog.cc:455] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 251861861371402797d330f9ecdd6143. Latest consensus state: current_term: 3 leader_uuid: "251861861371402797d330f9ecdd6143" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:13.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.955117 10334 sys_catalog.cc:458] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: This master's current role is: LEADER
15:06:13.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.965477 10338 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
15:06:13.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.975145 10338 catalog_manager.cc:1485] Initializing Kudu cluster ID...
15:06:13.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.979920 10338 catalog_manager.cc:1260] Loaded cluster ID: ef52103ef7db46f78a1ee3713871c7d3
15:06:13.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.980305 10338 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
15:06:13.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.986356 10338 catalog_manager.cc:1505] Loading token signing keys...
15:06:13.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.991329 10338 catalog_manager.cc:5910] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: Loaded TSK: 0
15:06:13.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:13.992643 10338 catalog_manager.cc:1515] Initializing in-progress tserver states...
15:06:14.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:14 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780774, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.107738 9339 init.cc:295] Successfully reacquired a new kerberos TGT
15:06:14.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:12.962435 10322 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:14.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:14.525424 10324 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:14.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:14.528111 10323 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1566 milliseconds
15:06:14.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.528239 10291 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:06:14.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.529642 10291 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.535171 10265 raft_consensus.cc:1270] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 3 FOLLOWER]: Refusing update from remote peer 251861861371402797d330f9ecdd6143: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
15:06:14.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.535590 10291 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:14.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.537102 10291 hybrid_clock.cc:648] HybridClock initialized: now 1736780774537039 us; error 66 us; skew 500 ppm
15:06:14.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.536861 10340 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
15:06:14.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.541745 10291 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:14.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:14 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780774, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:14.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.552147 10291 init.cc:377] Logged in from keytab as kudu/127.8.198.125@KRBTEST.COM (short username kudu)
15:06:14.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:14.571050 10135 consensus_peers.cc:487] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 -> Peer 712570e4ca7d40e9b588643a35fd6197 (127.8.198.125:33167): Couldn't send request to peer 712570e4ca7d40e9b588643a35fd6197. Status: Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
15:06:14.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.581292 10291 webserver.cc:458] Webserver started at http://127.8.198.125:37437/ using document root <none> and password file <none>
15:06:14.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.582657 10291 fs_manager.cc:362] Metadata directory not provided
15:06:14.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.582955 10291 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:14.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.583806 10342 sys_catalog.cc:455] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 251861861371402797d330f9ecdd6143. Latest consensus state: current_term: 3 leader_uuid: "251861861371402797d330f9ecdd6143" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:14.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.584529 10342 sys_catalog.cc:458] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: This master's current role is: FOLLOWER
15:06:14.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.602499 10342 sys_catalog.cc:455] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "251861861371402797d330f9ecdd6143" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:14.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.603443 10342 sys_catalog.cc:458] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: This master's current role is: FOLLOWER
15:06:14.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.607885 10291 fs_manager.cc:714] Time spent opening directory manager: real 0.021s user 0.005s sys 0.011s
15:06:14.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.608651 10349 sys_catalog.cc:455] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "251861861371402797d330f9ecdd6143" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:14.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.609158 10349 sys_catalog.cc:458] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: This master's current role is: LEADER
15:06:14.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.614420 10355 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:14.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.615501 10291 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
15:06:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.615839 10291 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data,/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/wal
15:06:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "712570e4ca7d40e9b588643a35fd6197"
15:06:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:48 on dist-test-slave-s7w9"
15:06:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.618115 10291 fs_report.cc:389] FS layout report
15:06:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/wal
15:06:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/wal
15:06:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data/data
15:06:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:14.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:14.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:14.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:14.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:14.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:14.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:14.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.665693 10291 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:14.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.672683 10291 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:14.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.673254 10291 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:14.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.773052 10291 rpc_server.cc:307] RPC server started. Bound to: 127.8.198.125:33167
15:06:14.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.773231 10407 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.198.125:33167 every 8 connection(s)
15:06:14.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.775902 10291 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data/info.pb
15:06:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.784276 8985 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu as pid 10291
15:06:14.785 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:14.785 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.198.126:39143(127.8.198.126:39143)] connecting to peer
15:06:14.787 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.8.198.126:39143(127.8.198.126:39143)] Successfully connected to peer
15:06:14.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.790855 10408 sys_catalog.cc:263] Verifying existing consensus state
15:06:14.795 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.8.198.126
15:06:14.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.799865 10408 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: Bootstrap starting.
15:06:14.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:14 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780768, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.8.198.126@KRBTEST.COM
15:06:14.856 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xc8965c35, L:/127.0.0.1:51910 - R:/127.8.198.126:39143] using SASL/GSSAPI
15:06:14.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.882951 10408 log.cc:826] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: Log is configured to *not* fsync() on all Append() calls
15:06:14.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.902858 10408 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:06:14.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.903784 10408 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: Bootstrap complete.
15:06:14.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.933714 10408 raft_consensus.cc:357] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:14.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.935986 10408 raft_consensus.cc:738] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 712570e4ca7d40e9b588643a35fd6197, State: Initialized, Role: FOLLOWER
15:06:14.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.936705 10408 consensus_queue.cc:260] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:14.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.939553 10413 sys_catalog.cc:455] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:14.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.940408 10413 sys_catalog.cc:458] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: This master's current role is: FOLLOWER
15:06:14.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.941300 10408 sys_catalog.cc:564] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: configured and running, proceeding with master startup.
15:06:14.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.980883 10424 catalog_manager.cc:1260] Loaded cluster ID: ef52103ef7db46f78a1ee3713871c7d3
15:06:14.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.981148 10424 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: loading cluster ID for follower catalog manager: success
15:06:14.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.989219 10424 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: acquiring CA information for follower catalog manager: success
15:06:14.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:14.998771 10424 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:06:15.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:15 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780774, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.125@KRBTEST.COM for kudu/127.8.198.126@KRBTEST.COM
15:06:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.035365 9747 heartbeater.cc:346] Connected to a master server at 127.8.198.125:33167
15:06:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.035624 9457 heartbeater.cc:346] Connected to a master server at 127.8.198.125:33167
15:06:15.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.047896 10372 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "a6a098cf6c944ee09b741bae26efb0aa" instance_seqno: 1736780758465558) as {username='kudu', principal='kudu/127.8.198.67@KRBTEST.COM'} at 127.8.198.67:37185; Asking this server to re-register.
15:06:15.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.049749 10371 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "4af75b1b5ad4450385a38dc956ee3969" instance_seqno: 1736780754350460) as {username='kudu', principal='kudu/127.8.198.65@KRBTEST.COM'} at 127.8.198.65:44339; Asking this server to re-register.
15:06:15.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.056178 9747 heartbeater.cc:463] Registering TS with master...
15:06:15.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.056275 9457 heartbeater.cc:463] Registering TS with master...
15:06:15.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.060297 10371 ts_manager.cc:194] Registered new tserver with Master: a6a098cf6c944ee09b741bae26efb0aa (127.8.198.67:45947)
15:06:15.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.062258 10372 ts_manager.cc:194] Registered new tserver with Master: 4af75b1b5ad4450385a38dc956ee3969 (127.8.198.65:42545)
15:06:15.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.070133 10382 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [term 2 FOLLOWER]: Advancing to term 3
15:06:15.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.106700 10434 sys_catalog.cc:455] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "251861861371402797d330f9ecdd6143" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:15.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.108047 10434 sys_catalog.cc:458] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: This master's current role is: FOLLOWER
15:06:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.110061 10413 sys_catalog.cc:455] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 251861861371402797d330f9ecdd6143. Latest consensus state: current_term: 3 leader_uuid: "251861861371402797d330f9ecdd6143" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.110541 10413 sys_catalog.cc:458] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: This master's current role is: FOLLOWER
15:06:15.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.522385 10349 sys_catalog.cc:455] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "251861861371402797d330f9ecdd6143" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:15.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.523440 10349 sys_catalog.cc:458] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: This master's current role is: LEADER
15:06:15.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.711498 9605 heartbeater.cc:502] Master 127.8.198.126:39143 was elected leader, sending a full tablet report...
15:06:15.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.799090 9602 heartbeater.cc:346] Connected to a master server at 127.8.198.125:33167
15:06:15.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.801752 10372 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "314efce02ef6446094dd961b666f16b4" instance_seqno: 1736780756363637) as {username='kudu', principal='kudu/127.8.198.66@KRBTEST.COM'} at 127.8.198.66:35823; Asking this server to re-register.
15:06:15.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.802882 9602 heartbeater.cc:463] Registering TS with master...
15:06:15.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.805874 10372 ts_manager.cc:194] Registered new tserver with Master: 314efce02ef6446094dd961b666f16b4 (127.8.198.66:37253)
15:06:15.862 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:06:15.862 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:06:15.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.873633 9748 heartbeater.cc:502] Master 127.8.198.126:39143 was elected leader, sending a full tablet report...
15:06:15.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:15 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780775, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:15.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:06:15.900 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:15.901 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5cc
15:06:15.902 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:06:15.903 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.198.126:39143
15:06:15.903 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
15:06:15.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.903761 8985 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu with pid 10106
15:06:15.945 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.8.198.126:39143(127.8.198.126:39143)] handling channelInactive
15:06:15.945 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.8.198.126:39143(127.8.198.126:39143)] cleaning up while in state READY due to: connection closed
15:06:15.947 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:15.947 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.198.124:34423
15:06:15.947 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
15:06:15.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.947902 8985 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu with pid 10186
15:06:15.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:15.986377 9344 connection.cc:537] client connection to 127.8.198.124:34423 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
15:06:15.987 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:15.988 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.8.198.125:33167
15:06:15.988 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
15:06:15.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:15.988653 8985 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu with pid 10291
15:06:16.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:16.004084 9458 heartbeater.cc:502] Master 127.8.198.126:39143 was elected leader, sending a full tablet report...
15:06:16.014 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:16.014 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.198.126:39143
15:06:16.014 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
15:06:16.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:16.016240 8985 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:06:16.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/wal
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/logs
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data/info.pb
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.126:39143
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.126
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44225
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.126.keytab
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.126
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:06:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:06:16.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:16 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780776, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:16.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:16.142596 9484 init.cc:295] Successfully reacquired a new kerberos TGT
15:06:16.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:16.375231 10444 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:16.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:16.375871 10444 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:16.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:16.376545 10444 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:16.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:16.415059 10444 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:06:16.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:16.415319 10444 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:06:16.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:16.415701 10444 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:16.416008 10444 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:06:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:16.416306 10444 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:06:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:16.416586 10444 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:06:16.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:16.416810 10444 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41287/jwks.json
15:06:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:16.453882 10444 master_runner.cc:386] Master server non-default flags:
15:06:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:06:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data
15:06:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/wal
15:06:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:06:16.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:16.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:06:16.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.126.keytab
15:06:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.126
15:06:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.126:39143
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data/info.pb
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.126
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44225
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/logs
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:16.455530 10444 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:16.456794 10444 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:16.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:16.472112 10452 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:17.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:17.218264 8829 debug-util.cc:398] Leaking SignalData structure 0x7b08000b10a0 after lost signal to thread 8705
15:06:17.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:17.219048 8829 debug-util.cc:398] Leaking SignalData structure 0x7b08000bacc0 after lost signal to thread 8832
15:06:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:16.472498 10451 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:16.474138 10444 server_base.cc:1034] running on GCE node
15:06:17.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:16.474756 10454 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:17.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:17.704361 10444 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:17.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:17.707777 10444 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:17.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:17.709350 10444 hybrid_clock.cc:648] HybridClock initialized: now 1736780777709272 us; error 66 us; skew 500 ppm
15:06:17.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:17.710490 10444 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:17.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:17 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780777, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:17.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:17.718077 10444 init.cc:377] Logged in from keytab as kudu/127.8.198.126@KRBTEST.COM (short username kudu)
15:06:17.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:17.736697 10444 webserver.cc:458] Webserver started at http://127.8.198.126:44225/ using document root <none> and password file <none>
15:06:17.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:17.738037 10444 fs_manager.cc:362] Metadata directory not provided
15:06:17.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:17.738314 10444 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:17.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:17.749696 10444 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.006s sys 0.001s
15:06:17.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:17.755545 10467 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:17.759792 10444 fs_manager.cc:730] Time spent opening block manager: real 0.007s user 0.005s sys 0.000s
15:06:17.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:17.760118 10444 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data,/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/wal
15:06:17.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "251861861371402797d330f9ecdd6143"
15:06:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:46 on dist-test-slave-s7w9"
15:06:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:17.762563 10444 fs_report.cc:389] FS layout report
15:06:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/wal
15:06:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/wal
15:06:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data/data
15:06:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:17.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:17.830441 10444 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:17.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:17.837391 10444 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:17.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:17.837852 10444 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:17.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:17.948608 10444 rpc_server.cc:307] RPC server started. Bound to: 127.8.198.126:39143
15:06:17.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:17.948746 10519 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.198.126:39143 every 8 connection(s)
15:06:17.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:17.951428 10444 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-0/data/info.pb
15:06:17.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:17.959998 8985 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu as pid 10444
15:06:17.960 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:17.960 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.198.124:34423
15:06:17.961 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:17.962704 8985 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/wal
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/logs
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data/info.pb
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.124:34423
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.124
15:06:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36391
15:06:17.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.124.keytab
15:06:17.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.124
15:06:17.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:17.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:06:17.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:06:17.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:06:17.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:17.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:17.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:17.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:17.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:06:17.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:17.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:06:17.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:06:17.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:17.966779 10520 sys_catalog.cc:263] Verifying existing consensus state
15:06:17.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:17.975414 10520 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: Bootstrap starting.
15:06:18.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.047319 10520 log.cc:826] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: Log is configured to *not* fsync() on all Append() calls
15:06:18.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:18 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780774, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.65@KRBTEST.COM for kudu/127.8.198.126@KRBTEST.COM
15:06:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.070528 9458 heartbeater.cc:346] Connected to a master server at 127.8.198.126:39143
15:06:18.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.071066 9458 heartbeater.cc:502] Master 127.8.198.126:39143 was elected leader, sending a full tablet report...
15:06:18.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.086311 10520 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:06:18.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.087402 10520 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: Bootstrap complete.
15:06:18.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.122781 10520 raft_consensus.cc:357] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:18.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.126387 10520 raft_consensus.cc:738] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 251861861371402797d330f9ecdd6143, State: Initialized, Role: FOLLOWER
15:06:18.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.127497 10520 consensus_queue.cc:260] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:18.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:18 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780778, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:18.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.134698 9629 init.cc:295] Successfully reacquired a new kerberos TGT
15:06:18.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.142288 10520 sys_catalog.cc:564] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: configured and running, proceeding with master startup.
15:06:18.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.141170 10527 sys_catalog.cc:455] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:18.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.144134 10527 sys_catalog.cc:458] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: This master's current role is: FOLLOWER
15:06:18.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.200068 10538 catalog_manager.cc:1260] Loaded cluster ID: ef52103ef7db46f78a1ee3713871c7d3
15:06:18.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.207882 10538 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: loading cluster ID for follower catalog manager: success
15:06:18.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:18 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780777, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.126@KRBTEST.COM for kudu/127.8.198.126@KRBTEST.COM
15:06:18.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.216742 10538 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: acquiring CA information for follower catalog manager: success
15:06:18.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.222342 10538 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:06:18.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.233299 10444 master_runner.cc:186] Error getting master registration for 127.8.198.125:33167: Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111),
15:06:18.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.269620 10444 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:06:18.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:18.486054 10522 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:18.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:18.486882 10522 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:18.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:18.488075 10522 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:18.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:18.570256 10522 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:06:18.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:18.570830 10522 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:06:18.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:18.571558 10522 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:18.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:18.572180 10522 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:06:18.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:18.572667 10522 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:06:18.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:18.573109 10522 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:06:18.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:18.573486 10522 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41287/jwks.json
15:06:18.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.640293 10522 master_runner.cc:386] Master server non-default flags:
15:06:18.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:18.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:06:18.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:18.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/wal
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.124.keytab
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.124
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.124:34423
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data/info.pb
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.124
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36391
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/logs
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:18.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:18.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:18.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:06:18.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:06:18.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:06:18.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:18.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:06:18.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:06:18.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:18.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.643442 10522 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:18.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.645270 10522 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:18.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:18.667455 10546 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:18.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.792359 9605 heartbeater.cc:346] Connected to a master server at 127.8.198.126:39143
15:06:18.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.808281 10484 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "314efce02ef6446094dd961b666f16b4" instance_seqno: 1736780756363637) as {username='kudu', principal='kudu/127.8.198.66@KRBTEST.COM'} at 127.8.198.66:34903; Asking this server to re-register.
15:06:18.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.811399 9605 heartbeater.cc:463] Registering TS with master...
15:06:18.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.815426 10484 ts_manager.cc:194] Registered new tserver with Master: 314efce02ef6446094dd961b666f16b4 (127.8.198.66:37253)
15:06:18.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.933138 9748 heartbeater.cc:346] Connected to a master server at 127.8.198.126:39143
15:06:18.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.937826 10484 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "a6a098cf6c944ee09b741bae26efb0aa" instance_seqno: 1736780758465558) as {username='kudu', principal='kudu/127.8.198.67@KRBTEST.COM'} at 127.8.198.67:57333; Asking this server to re-register.
15:06:18.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.940441 9748 heartbeater.cc:463] Registering TS with master...
15:06:18.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:18.943641 10484 ts_manager.cc:194] Registered new tserver with Master: a6a098cf6c944ee09b741bae26efb0aa (127.8.198.67:45947)
15:06:19.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:19.079067 9458 heartbeater.cc:502] Master 127.8.198.126:39143 was elected leader, sending a full tablet report...
15:06:19.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:19.082437 10484 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "4af75b1b5ad4450385a38dc956ee3969" instance_seqno: 1736780754350460) as {username='kudu', principal='kudu/127.8.198.65@KRBTEST.COM'} at 127.8.198.65:32789; Asking this server to re-register.
15:06:19.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:19.085057 9458 heartbeater.cc:463] Registering TS with master...
15:06:19.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:19.088266 10484 ts_manager.cc:194] Registered new tserver with Master: 4af75b1b5ad4450385a38dc956ee3969 (127.8.198.65:42545)
15:06:19.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:19.908417 10558 raft_consensus.cc:491] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:06:19.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:19.911885 10558 raft_consensus.cc:513] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:19.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:19.946085 10558 leader_election.cc:290] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 712570e4ca7d40e9b588643a35fd6197 (127.8.198.125:33167), a32bb1830f4f4f458a1ad99a5ed3122f (127.8.198.124:34423)
15:06:19.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:19.972232 10468 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.198.124:34423: connect: Connection refused (error 111)
15:06:19.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:19.992174 10470 leader_election.cc:336] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 712570e4ca7d40e9b588643a35fd6197 (127.8.198.125:33167): Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111)
15:06:20.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:20.000747 10468 leader_election.cc:336] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer a32bb1830f4f4f458a1ad99a5ed3122f (127.8.198.124:34423): Network error: Client connection negotiation failed: client connection to 127.8.198.124:34423: connect: Connection refused (error 111)
15:06:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.001772 10468 leader_election.cc:304] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 251861861371402797d330f9ecdd6143; no voters: 712570e4ca7d40e9b588643a35fd6197, a32bb1830f4f4f458a1ad99a5ed3122f
15:06:20.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.003166 10558 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
15:06:20.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:18.669073 10547 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:20.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:20.069020 10545 debug-util.cc:398] Leaking SignalData structure 0x7b080001cac0 after lost signal to thread 10522
15:06:20.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:20.758950 10522 thread.cc:635] OpenStack (cloud detector) Time spent creating pthread: real 2.089s user 0.634s sys 1.068s
15:06:20.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:20.761181 10522 thread.cc:601] OpenStack (cloud detector) Time spent starting thread: real 2.092s user 0.634s sys 1.068s
15:06:20.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:20.762602 10549 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:20.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:20.763852 10548 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2093 milliseconds
15:06:20.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.763927 10522 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:06:20.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.765400 10522 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.768442 10522 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:20.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.769884 10522 hybrid_clock.cc:648] HybridClock initialized: now 1736780780769827 us; error 60 us; skew 500 ppm
15:06:20.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.770992 10522 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:20.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:20 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780780, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:20.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.778383 10522 init.cc:377] Logged in from keytab as kudu/127.8.198.124@KRBTEST.COM (short username kudu)
15:06:20.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.786042 10522 webserver.cc:458] Webserver started at http://127.8.198.124:36391/ using document root <none> and password file <none>
15:06:20.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.787263 10522 fs_manager.cc:362] Metadata directory not provided
15:06:20.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.787536 10522 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:20.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.798015 10522 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.004s sys 0.003s
15:06:20.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.803294 10573 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:20.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.804570 10522 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
15:06:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.804987 10522 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data,/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/wal
15:06:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a32bb1830f4f4f458a1ad99a5ed3122f"
15:06:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:50 on dist-test-slave-s7w9"
15:06:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.808295 10522 fs_report.cc:389] FS layout report
15:06:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/wal
15:06:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/wal
15:06:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data/data
15:06:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.870321 10522 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:20.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.876801 10522 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:20.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.877296 10522 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.962136 10522 rpc_server.cc:307] RPC server started. Bound to: 127.8.198.124:34423
15:06:20.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.962289 10625 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.198.124:34423 every 8 connection(s)
15:06:20.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.974177 10522 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-2/data/info.pb
15:06:20.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.982648 8985 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu as pid 10522
15:06:20.983 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:20.983 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.8.198.125:33167
15:06:20.984 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
15:06:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.986299 8985 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:06:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu
15:06:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/wal
15:06:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data
15:06:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/logs
15:06:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data/info.pb
15:06:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:06:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:20.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.125:33167
15:06:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.125
15:06:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37437
15:06:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.125.keytab
15:06:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.125
15:06:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:06:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:06:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:06:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:06:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:06:20.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:06:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:20.994095 10628 sys_catalog.cc:263] Verifying existing consensus state
15:06:20.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:20 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780778, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.67@KRBTEST.COM for kudu/127.8.198.124@KRBTEST.COM
15:06:21.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.002610 10628 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: Bootstrap starting.
15:06:21.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.014704 9746 heartbeater.cc:346] Connected to a master server at 127.8.198.124:34423
15:06:21.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.058684 10628 log.cc:826] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: Log is configured to *not* fsync() on all Append() calls
15:06:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:21 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780774, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.65@KRBTEST.COM for kudu/127.8.198.124@KRBTEST.COM
15:06:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.099198 10628 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:06:21.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.100281 10628 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: Bootstrap complete.
15:06:21.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.107765 9456 heartbeater.cc:346] Connected to a master server at 127.8.198.124:34423
15:06:21.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.130800 10628 raft_consensus.cc:357] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:21.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.134258 10628 raft_consensus.cc:738] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: a32bb1830f4f4f458a1ad99a5ed3122f, State: Initialized, Role: FOLLOWER
15:06:21.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.135269 10628 consensus_queue.cc:260] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:21.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.151098 10634 sys_catalog.cc:455] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:21.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.152016 10634 sys_catalog.cc:458] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: This master's current role is: FOLLOWER
15:06:21.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.153599 10628 sys_catalog.cc:564] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: configured and running, proceeding with master startup.
15:06:21.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.197793 10645 catalog_manager.cc:1260] Loaded cluster ID: ef52103ef7db46f78a1ee3713871c7d3
15:06:21.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.198323 10645 catalog_manager.cc:1553] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: loading cluster ID for follower catalog manager: success
15:06:21.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.205499 10645 catalog_manager.cc:1575] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: acquiring CA information for follower catalog manager: success
15:06:21.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.218643 10645 catalog_manager.cc:1603] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:06:21.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.233911 10522 master_runner.cc:186] Error getting master registration for 127.8.198.125:33167: Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111),
15:06:21.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.265093 10522 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:06:21.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:21.450628 10630 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:21.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:21.451390 10630 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:21.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:21.452492 10630 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:21.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:21.509581 10630 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:06:21.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:21.510159 10630 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:06:21.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:21.510773 10630 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:21.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:21.511225 10630 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:06:21.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:21.511804 10630 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:06:21.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:21.512248 10630 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:06:21.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:21.512668 10630 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41287/jwks.json
15:06:21.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.593971 10630 master_runner.cc:386] Master server non-default flags:
15:06:21.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:21.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.198.84:36937
15:06:21.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:21.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:21.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data
15:06:21.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/wal
15:06:21.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.198.126:39143,127.8.198.125:33167,127.8.198.124:34423
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/krb5kdc/kudu_127.8.198.125.keytab
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.8.198.125
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.198.125:33167
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41287/jwks.json
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data/info.pb
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.198.125
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37437
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/logs
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Jan 2025 14:43:21 UTC on a633016ff5c9
15:06:21.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4160
15:06:21.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:21.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.595844 10630 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:21.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.597363 10630 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:21.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:21.617377 10654 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:21.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.931823 9601 heartbeater.cc:346] Connected to a master server at 127.8.198.124:34423
15:06:21.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.941047 10591 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "314efce02ef6446094dd961b666f16b4" instance_seqno: 1736780756363637) as {username='kudu', principal='kudu/127.8.198.66@KRBTEST.COM'} at 127.8.198.66:48869; Asking this server to re-register.
15:06:21.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.942615 9601 heartbeater.cc:463] Registering TS with master...
15:06:21.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:21.946583 10591 ts_manager.cc:194] Registered new tserver with Master: 314efce02ef6446094dd961b666f16b4 (127.8.198.66:37253)
15:06:22.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.020432 10591 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "a6a098cf6c944ee09b741bae26efb0aa" instance_seqno: 1736780758465558) as {username='kudu', principal='kudu/127.8.198.67@KRBTEST.COM'} at 127.8.198.67:58873; Asking this server to re-register.
15:06:22.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.022039 9746 heartbeater.cc:463] Registering TS with master...
15:06:22.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.025075 10591 ts_manager.cc:194] Registered new tserver with Master: a6a098cf6c944ee09b741bae26efb0aa (127.8.198.67:45947)
15:06:22.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.113602 10591 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "4af75b1b5ad4450385a38dc956ee3969" instance_seqno: 1736780754350460) as {username='kudu', principal='kudu/127.8.198.65@KRBTEST.COM'} at 127.8.198.65:52443; Asking this server to re-register.
15:06:22.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.115303 9456 heartbeater.cc:463] Registering TS with master...
15:06:22.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.118136 10591 ts_manager.cc:194] Registered new tserver with Master: 4af75b1b5ad4450385a38dc956ee3969 (127.8.198.65:42545)
15:06:22.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.158756 10668 raft_consensus.cc:491] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:06:22.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.159339 10668 raft_consensus.cc:513] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:22.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.161626 10668 leader_election.cc:290] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 712570e4ca7d40e9b588643a35fd6197 (127.8.198.125:33167), a32bb1830f4f4f458a1ad99a5ed3122f (127.8.198.124:34423)
15:06:22.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:22.174907 10470 leader_election.cc:336] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 712570e4ca7d40e9b588643a35fd6197 (127.8.198.125:33167): Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111)
15:06:22.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.186311 10601 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "251861861371402797d330f9ecdd6143" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" is_pre_election: true
15:06:22.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.187520 10601 raft_consensus.cc:2463] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 251861861371402797d330f9ecdd6143 in term 3.
15:06:22.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.190002 10468 leader_election.cc:304] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 251861861371402797d330f9ecdd6143, a32bb1830f4f4f458a1ad99a5ed3122f; no voters: 712570e4ca7d40e9b588643a35fd6197
15:06:22.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.191358 10668 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 3 FOLLOWER]: Leader pre-election won for term 4
15:06:22.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.191813 10668 raft_consensus.cc:491] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:06:22.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.192214 10668 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 3 FOLLOWER]: Advancing to term 4
15:06:22.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.206387 10668 raft_consensus.cc:513] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:22.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.209915 10601 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "251861861371402797d330f9ecdd6143" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f"
15:06:22.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.210475 10601 raft_consensus.cc:3054] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 3 FOLLOWER]: Advancing to term 4
15:06:22.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.215404 10668 leader_election.cc:290] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 4 election: Requested vote from peers 712570e4ca7d40e9b588643a35fd6197 (127.8.198.125:33167), a32bb1830f4f4f458a1ad99a5ed3122f (127.8.198.124:34423)
15:06:22.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:22.216260 10470 leader_election.cc:336] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 712570e4ca7d40e9b588643a35fd6197 (127.8.198.125:33167): Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111)
15:06:22.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.230203 10601 raft_consensus.cc:2463] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 251861861371402797d330f9ecdd6143 in term 4.
15:06:22.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.243407 10468 leader_election.cc:304] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 251861861371402797d330f9ecdd6143, a32bb1830f4f4f458a1ad99a5ed3122f; no voters: 712570e4ca7d40e9b588643a35fd6197
15:06:22.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.245056 10668 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 4 FOLLOWER]: Leader election won for term 4
15:06:22.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.262074 10668 raft_consensus.cc:695] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [term 4 LEADER]: Becoming Leader. State: Replica: 251861861371402797d330f9ecdd6143, State: Running, Role: LEADER
15:06:22.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.263409 10668 consensus_queue.cc:237] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:22.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.282291 10672 sys_catalog.cc:455] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 251861861371402797d330f9ecdd6143. Latest consensus state: current_term: 4 leader_uuid: "251861861371402797d330f9ecdd6143" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:22.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.283349 10672 sys_catalog.cc:458] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: This master's current role is: LEADER
15:06:22.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.292528 10674 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
15:06:22.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.314328 10674 catalog_manager.cc:1485] Initializing Kudu cluster ID...
15:06:22.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.335515 10674 catalog_manager.cc:1260] Loaded cluster ID: ef52103ef7db46f78a1ee3713871c7d3
15:06:22.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.335852 10674 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
15:06:22.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.350795 10674 catalog_manager.cc:1505] Loading token signing keys...
15:06:22.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.365200 10674 catalog_manager.cc:5910] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143: Loaded TSK: 0
15:06:22.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.366958 10674 catalog_manager.cc:1515] Initializing in-progress tserver states...
15:06:22.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:22.576668 8338 debug-util.cc:398] Leaking SignalData structure 0x7b08000776a0 after lost signal to thread 8275
15:06:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:22.578214 8338 debug-util.cc:398] Leaking SignalData structure 0x7b08000a8120 after lost signal to thread 8341
15:06:22.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:22.815943 10470 consensus_peers.cc:487] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 -> Peer 712570e4ca7d40e9b588643a35fd6197 (127.8.198.125:33167): Couldn't send request to peer 712570e4ca7d40e9b588643a35fd6197. Status: Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
15:06:22.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.827632 10601 raft_consensus.cc:1270] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [term 4 FOLLOWER]: Refusing update from remote peer 251861861371402797d330f9ecdd6143: Log matching property violated. Preceding OpId in replica: term: 3 index: 6. Preceding OpId from leader: term: 4 index: 7. (index mismatch)
15:06:22.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.830190 10672 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.001s
15:06:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.863500 10678 sys_catalog.cc:455] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 251861861371402797d330f9ecdd6143. Latest consensus state: current_term: 4 leader_uuid: "251861861371402797d330f9ecdd6143" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.864527 10678 sys_catalog.cc:458] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: This master's current role is: FOLLOWER
15:06:22.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.924614 10672 sys_catalog.cc:455] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "251861861371402797d330f9ecdd6143" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:22.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.925614 10672 sys_catalog.cc:458] T 00000000000000000000000000000000 P 251861861371402797d330f9ecdd6143 [sys.catalog]: This master's current role is: LEADER
15:06:22.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.929157 10678 sys_catalog.cc:455] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "251861861371402797d330f9ecdd6143" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:22.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:22.930183 10678 sys_catalog.cc:458] T 00000000000000000000000000000000 P a32bb1830f4f4f458a1ad99a5ed3122f [sys.catalog]: This master's current role is: FOLLOWER
15:06:23.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:23.274951 10444 master_runner.cc:186] Error getting master registration for 127.8.198.125:33167: Network error: Client connection negotiation failed: client connection to 127.8.198.125:33167: connect: Connection refused (error 111),
15:06:23.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:23.275381 10444 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:06:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:23.924028 9605 heartbeater.cc:502] Master 127.8.198.126:39143 was elected leader, sending a full tablet report...
15:06:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:23.974980 9748 heartbeater.cc:502] Master 127.8.198.126:39143 was elected leader, sending a full tablet report...
15:06:24.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:24 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780784, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:24.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:24.111078 9339 init.cc:295] Successfully reacquired a new kerberos TGT
15:06:24.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:24.155874 9458 heartbeater.cc:502] Master 127.8.198.126:39143 was elected leader, sending a full tablet report...
15:06:24.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:23.013105 10653 debug-util.cc:398] Leaking SignalData structure 0x7b080001cac0 after lost signal to thread 10630
15:06:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:24.815979 10653 kernel_stack_watchdog.cc:198] Thread 10630 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:636 for 390ms:
15:06:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:06:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:06:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:06:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:06:24.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:21.618780 10655 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:24.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:24.817585 10656 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3197 milliseconds
15:06:24.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:24.818794 10630 thread.cc:635] OpenStack (cloud detector) Time spent creating pthread: real 3.198s user 0.919s sys 1.614s
15:06:24.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:24.819131 10630 thread.cc:601] OpenStack (cloud detector) Time spent starting thread: real 3.199s user 0.919s sys 1.614s
15:06:24.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:24.819371 10658 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:24.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:24.819418 10630 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:06:24.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:24.827406 10630 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:24.852336 10630 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:24.855948 10630 hybrid_clock.cc:648] HybridClock initialized: now 1736780784855890 us; error 55 us; skew 500 ppm
15:06:24.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:24.857259 10630 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:24.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:24 dist-test-slave-s7w9 krb5kdc[8998](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780784, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:24.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:24.871001 10630 init.cc:377] Logged in from keytab as kudu/127.8.198.125@KRBTEST.COM (short username kudu)
15:06:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:24.902346 10630 webserver.cc:458] Webserver started at http://127.8.198.125:37437/ using document root <none> and password file <none>
15:06:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:24.903690 10630 fs_manager.cc:362] Metadata directory not provided
15:06:24.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:24.904012 10630 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:24.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:24.916813 10630 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.007s sys 0.000s
15:06:24.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:24.922900 10699 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:24.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:24.924213 10630 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
15:06:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:24.924638 10630 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data,/tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/wal
15:06:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "712570e4ca7d40e9b588643a35fd6197"
15:06:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-13 15:05:48 on dist-test-slave-s7w9"
15:06:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:24.928277 10630 fs_report.cc:389] FS layout report
15:06:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/wal
15:06:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/wal
15:06:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data/data
15:06:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:24.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:25.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.043519 10630 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:25.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.052505 10630 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:25.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.053086 10630 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:25.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.221797 10630 rpc_server.cc:307] RPC server started. Bound to: 127.8.198.125:33167
15:06:25.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.222272 10754 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.198.125:33167 every 8 connection(s)
15:06:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.226003 10630 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYvjSJN/test-tmp/mini-kudu-cluster1450648170588345549/master-1/data/info.pb
15:06:25.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.233384 8985 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu as pid 10630
15:06:25.235 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:25.235 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.8.198.126:39143(127.8.198.126:39143)] connecting to peer
15:06:25.238 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.8.198.126:39143(127.8.198.126:39143)] Successfully connected to peer
15:06:25.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.249046 10755 sys_catalog.cc:263] Verifying existing consensus state
15:06:25.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.8.198.126:39143(127.8.198.126:39143)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
15:06:25.253 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.8.198.126:39143(127.8.198.126:39143)] handling channelInactive
15:06:25.256 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.8.198.125:33167(127.8.198.125:33167)] disconnecting while in state TERMINATED
15:06:25.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.258442 10755 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: Bootstrap starting.
15:06:25.261 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.8.198.124:34423(127.8.198.124:34423)] disconnecting while in state TERMINATED
15:06:25.263 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.8.198.126:39143(127.8.198.126:39143)] disconnecting while in state TERMINATED
15:06:25.267 [DEBUG - Time-limited test] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:06:25.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250113 15:06:25.272102 10757 negotiation.cc:337] Failed RPC negotiation. Trace:
15:06:25.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:06:25.240327 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:60926 (local address 127.8.198.126:39143)
15:06:25.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:06:25.251273 (+ 10946us) server_negotiation.cc:206] Beginning negotiation
15:06:25.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:06:25.251300 (+ 27us) server_negotiation.cc:399] Waiting for connection header
15:06:25.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:06:25.251352 (+ 52us) server_negotiation.cc:407] Connection header received
15:06:25.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:06:25.251500 (+ 148us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
15:06:25.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:06:25.251524 (+ 24us) server_negotiation.cc:461] Received NEGOTIATE request from client
15:06:25.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:06:25.251744 (+ 220us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
15:06:25.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:06:25.251944 (+ 200us) server_negotiation.cc:226] Negotiated authn=SASL
15:06:25.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0113 15:06:25.265879 (+ 13935us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:60926: BlockingRecv error: recv got EOF from 127.0.0.1:60926 (error 108)
15:06:25.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":10221,"thread_start_us":9927,"threads_started":1}
15:06:25.276 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:06:25.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.303068 8985 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu with pid 9297
15:06:25.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 13 15:06:25 dist-test-slave-s7w9 krb5kdc[8998](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736780777, etypes {rep=17 tkt=17 ses=17}, kudu/127.8.198.126@KRBTEST.COM for kudu/127.8.198.125@KRBTEST.COM
15:06:25.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.325225 10755 log.cc:826] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: Log is configured to *not* fsync() on all Append() calls
15:06:25.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.350780 8985 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu with pid 9468
15:06:25.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.368903 10755 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:06:25.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.369918 10755 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197: Bootstrap complete.
15:06:25.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.405462 8985 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu with pid 9613
15:06:25.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.403220 10755 raft_consensus.cc:357] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:25.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.408346 10755 raft_consensus.cc:738] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 712570e4ca7d40e9b588643a35fd6197, State: Initialized, Role: FOLLOWER
15:06:25.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.409319 10755 consensus_queue.cc:260] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } }
15:06:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.439841 10765 sys_catalog.cc:455] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "251861861371402797d330f9ecdd6143" member_type: VOTER last_known_addr { host: "127.8.198.126" port: 39143 } } peers { permanent_uuid: "712570e4ca7d40e9b588643a35fd6197" member_type: VOTER last_known_addr { host: "127.8.198.125" port: 33167 } } peers { permanent_uuid: "a32bb1830f4f4f458a1ad99a5ed3122f" member_type: VOTER last_known_addr { host: "127.8.198.124" port: 34423 } } }
15:06:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.441547 10765 sys_catalog.cc:458] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: This master's current role is: FOLLOWER
15:06:25.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.442211 10728 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [term 3 FOLLOWER]: Advancing to term 4
15:06:25.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.452028 10755 sys_catalog.cc:564] T 00000000000000000000000000000000 P 712570e4ca7d40e9b588643a35fd6197 [sys.catalog]: configured and running, proceeding with master startup.
15:06:25.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.455914 8985 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu with pid 10444
15:06:25.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.499536 8985 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu with pid 10630
15:06:25.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:25.526167 8985 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu with pid 10522
15:06:26.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-13T15:06:26Z chronyd exiting
15:06:27.775 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:06:27.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:27.776381 8262 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu with pid 8542
15:06:27.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:27.800346 8262 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu with pid 8698
15:06:27.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:27.827013 8262 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu with pid 8843
15:06:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:27.860098 8262 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu with pid 8274
15:06:27.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:27.891309 8262 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu with pid 8344
15:06:27.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250113 15:06:27.918792 8262 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskYvjSJN/build/tsan/bin/kudu with pid 8419
15:06:29.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-13T15:06:29Z chronyd exiting
15:06:30.059 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more