15:05:42.644 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
15:05:42.655 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskyZ3UZe/test-tmp
15:05:42.657 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:05:42.661 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskyZ3UZe/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:05:42.953 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
15:05:43.085 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:05:43.086 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:05:43.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:43.090271 13081 mini_oidc.cc:150] Starting JWKS server
15:05:43.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:43.093747 13081 webserver.cc:458] Webserver started at http://127.0.0.1:38759/ using document root <none> and password file <none>
15:05:43.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:43.094132 13081 mini_oidc.cc:186] Starting OIDC Discovery server
15:05:43.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:43.095816 13081 webserver.cc:458] Webserver started at http://127.0.0.1:45209/ using document root <none> and password file <none>
15:05:43.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-10T15:05:43Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
15:05:43.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-10T15:05:43Z Disabled control of system clock
15:05:43.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:43.162508 13081 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:05:43.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:05:43.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-0/wal
15:05:43.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-0/data
15:05:43.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:43.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:43.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:43.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:43.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:43.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:43.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-0/logs
15:05:43.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-0/data/info.pb
15:05:43.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:43.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:43.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:43.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:43.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:43.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:43.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:43.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:43.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:05:43.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:43.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:43.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:43.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.198.126:40351
15:05:43.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.198.126
15:05:43.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:43.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.198.84:32891
15:05:43.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:43.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:43.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:43.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:43.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.198.126:40351,127.12.198.125:37065,127.12.198.124:42251
15:05:43.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:43.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38759/jwks.json with env {}
15:05:43.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:43.458424 13093 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:43.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:43.459080 13093 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:43.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:43.459785 13093 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:43.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:43.489693 13093 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:05:43.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:43.490041 13093 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:43.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:43.490383 13093 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:05:43.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:43.490726 13093 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:05:43.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:43.491080 13093 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:05:43.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:43.491348 13093 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38759/jwks.json
15:05:43.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:43.525565 13093 master_runner.cc:386] Master server non-default flags:
15:05:43.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:43.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.198.84:32891
15:05:43.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:43.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:43.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-0/data
15:05:43.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-0/wal
15:05:43.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:43.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.198.126:40351,127.12.198.125:37065,127.12.198.124:42251
15:05:43.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:43.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:43.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:43.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.198.126:40351
15:05:43.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:43.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:43.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:43.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38759/jwks.json
15:05:43.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:43.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:43.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-0/data/info.pb
15:05:43.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.198.126
15:05:43.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:43.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:43.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:43.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:43.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:43.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:43.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-0/logs
15:05:43.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:43.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:43.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:43.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:05:43.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:43.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:43.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:43.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 14:43:22 UTC on a633016ff5c9
15:05:43.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4112
15:05:43.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:43.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:43.527302 13093 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:43.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:43.529505 13093 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:43.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:43.545909 13100 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:44.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:43.545948 13102 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:44.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:43.546736 13099 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:44.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:43.547535 13093 server_base.cc:1034] running on GCE node
15:05:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.728538 13093 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:44.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.731202 13093 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:44.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.732538 13093 hybrid_clock.cc:648] HybridClock initialized: now 1736521544732512 us; error 54 us; skew 500 ppm
15:05:44.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.733317 13093 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:44.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.744933 13093 webserver.cc:458] Webserver started at http://127.12.198.126:39227/ using document root <none> and password file <none>
15:05:44.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.745924 13093 fs_manager.cc:362] Metadata directory not provided
15:05:44.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.746116 13093 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:44.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.746510 13093 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:44.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.751385 13093 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-0/data/instance:
15:05:44.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "840a94c2474c4985848a471061fd1ba8"
15:05:44.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:05:44 on dist-test-slave-6prw"
15:05:44.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.752532 13093 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-0/wal/instance:
15:05:44.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "840a94c2474c4985848a471061fd1ba8"
15:05:44.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:05:44 on dist-test-slave-6prw"
15:05:44.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.759685 13093 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
15:05:44.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.765184 13109 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:44.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.766288 13093 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
15:05:44.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.766570 13093 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-0/data,/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-0/wal
15:05:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "840a94c2474c4985848a471061fd1ba8"
15:05:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:05:44 on dist-test-slave-6prw"
15:05:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.766871 13093 fs_report.cc:389] FS layout report
15:05:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-0/wal
15:05:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-0/wal
15:05:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-0/data/data
15:05:44.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:44.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:44.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:44.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:44.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:44.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:44.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:44.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:44.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:44.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:44.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:44.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.827008 13093 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:44.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.828496 13093 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:44.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.828900 13093 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:44.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.898744 13093 rpc_server.cc:307] RPC server started. Bound to: 127.12.198.126:40351
15:05:44.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.898811 13160 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.198.126:40351 every 8 connection(s)
15:05:44.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.901445 13093 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-0/data/info.pb
15:05:44.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.905045 13081 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu as pid 13093
15:05:44.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.905411 13081 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-0/wal/instance
15:05:44.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.907274 13161 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:05:44.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.908854 13081 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:05:44.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:05:44.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-1/wal
15:05:44.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-1/data
15:05:44.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:44.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:44.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:44.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:44.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:44.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:44.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-1/logs
15:05:44.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-1/data/info.pb
15:05:44.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:44.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:44.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:44.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:44.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:44.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:44.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:44.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:44.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:05:44.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:44.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:44.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:44.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.198.125:37065
15:05:44.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.198.125
15:05:44.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:44.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.198.84:32891
15:05:44.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:44.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:44.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:44.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:44.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.198.126:40351,127.12.198.125:37065,127.12.198.124:42251
15:05:44.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38759/jwks.json with env {}
15:05:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.920347 13161 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.198.126" port: 40351 } has no permanent_uuid. Determining permanent_uuid...
15:05:44.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:44.947216 13161 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } has no permanent_uuid. Determining permanent_uuid...
15:05:44.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:44.949445 13110 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.198.125:37065: connect: Connection refused (error 111)
15:05:44.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:44.953766 13161 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.198.125:37065: Network error: Client connection negotiation failed: client connection to 127.12.198.125:37065: connect: Connection refused (error 111)
15:05:45.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:45.003338 13161 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } attempt: 1
15:05:45.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:45.007350 13161 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.198.125:37065: Network error: Client connection negotiation failed: client connection to 127.12.198.125:37065: connect: Connection refused (error 111)
15:05:45.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:45.075775 13161 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } attempt: 2
15:05:45.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:45.079561 13161 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.198.125:37065: Network error: Client connection negotiation failed: client connection to 127.12.198.125:37065: connect: Connection refused (error 111)
15:05:45.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:45.170928 13161 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } attempt: 3
15:05:45.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:45.174679 13161 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.198.125:37065: Network error: Client connection negotiation failed: client connection to 127.12.198.125:37065: connect: Connection refused (error 111)
15:05:45.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:45.222168 13163 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:45.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:45.222769 13163 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:45.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:45.223528 13163 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:45.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:45.254185 13163 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:05:45.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:45.254528 13163 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:45.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:45.254734 13163 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:05:45.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:45.254913 13163 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:05:45.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:45.255095 13163 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:05:45.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:45.255244 13163 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38759/jwks.json
15:05:45.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:45.289119 13163 master_runner.cc:386] Master server non-default flags:
15:05:45.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:45.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.198.84:32891
15:05:45.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:45.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:45.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-1/data
15:05:45.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-1/wal
15:05:45.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:45.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.198.126:40351,127.12.198.125:37065,127.12.198.124:42251
15:05:45.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:45.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:45.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:45.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.198.125:37065
15:05:45.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:45.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:45.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:45.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38759/jwks.json
15:05:45.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:45.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:45.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-1/data/info.pb
15:05:45.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.198.125
15:05:45.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-1/logs
15:05:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:05:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 14:43:22 UTC on a633016ff5c9
15:05:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4112
15:05:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:45.290659 13163 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:45.292761 13163 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:45.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:45.309381 13175 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:45.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:45.318148 13161 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } attempt: 4
15:05:45.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:45.321847 13161 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.198.125:37065: Network error: Client connection negotiation failed: client connection to 127.12.198.125:37065: connect: Connection refused (error 111)
15:05:45.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:45.621264 13161 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } attempt: 5
15:05:45.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:45.626751 13161 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.198.125:37065: Network error: Client connection negotiation failed: client connection to 127.12.198.125:37065: connect: Connection refused (error 111)
15:05:46.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.174150 13161 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } attempt: 6
15:05:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:46.182204 13161 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.198.125:37065: Network error: Client connection negotiation failed: client connection to 127.12.198.125:37065: connect: Connection refused (error 111)
15:05:46.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:45.309847 13172 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:46.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:45.311152 13163 server_base.cc:1034] running on GCE node
15:05:46.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:45.310691 13173 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:46.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.493451 13163 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:46.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.496593 13163 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:46.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.498005 13163 hybrid_clock.cc:648] HybridClock initialized: now 1736521546497954 us; error 59 us; skew 500 ppm
15:05:46.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.499159 13163 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:46.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.505934 13163 webserver.cc:458] Webserver started at http://127.12.198.125:33873/ using document root <none> and password file <none>
15:05:46.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.507041 13163 fs_manager.cc:362] Metadata directory not provided
15:05:46.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.507323 13163 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:46.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.507876 13163 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:46.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.513077 13163 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-1/data/instance:
15:05:46.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86515feaf4ad4225bc17008d28478488"
15:05:46.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:05:46 on dist-test-slave-6prw"
15:05:46.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.514305 13163 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-1/wal/instance:
15:05:46.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86515feaf4ad4225bc17008d28478488"
15:05:46.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:05:46 on dist-test-slave-6prw"
15:05:46.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.521396 13163 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
15:05:46.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.526819 13184 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:46.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.527886 13163 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
15:05:46.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.528187 13163 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-1/data,/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-1/wal
15:05:46.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86515feaf4ad4225bc17008d28478488"
15:05:46.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:05:46 on dist-test-slave-6prw"
15:05:46.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.528548 13163 fs_report.cc:389] FS layout report
15:05:46.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:46.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-1/wal
15:05:46.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-1/wal
15:05:46.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-1/data/data
15:05:46.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:46.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:46.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:46.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:46.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:46.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:46.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:46.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:46.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:46.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:46.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:46.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.614497 13163 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:46.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.615913 13163 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:46.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.616377 13163 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:46.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.686141 13163 rpc_server.cc:307] RPC server started. Bound to: 127.12.198.125:37065
15:05:46.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.686228 13235 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.198.125:37065 every 8 connection(s)
15:05:46.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.689172 13163 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-1/data/info.pb
15:05:46.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.692286 13081 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu as pid 13163
15:05:46.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.692770 13081 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-1/wal/instance
15:05:46.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.695045 13236 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:05:46.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.695773 13081 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:05:46.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:05:46.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-2/wal
15:05:46.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-2/data
15:05:46.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:46.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:46.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:46.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:46.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:46.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:46.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-2/logs
15:05:46.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-2/data/info.pb
15:05:46.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:46.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:46.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:46.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:46.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:46.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:46.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:46.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:46.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:05:46.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:46.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:46.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:46.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.198.124:42251
15:05:46.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.198.124
15:05:46.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:46.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.198.84:32891
15:05:46.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:46.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:46.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:46.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:46.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.198.126:40351,127.12.198.125:37065,127.12.198.124:42251
15:05:46.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:46.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38759/jwks.json with env {}
15:05:46.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.708736 13236 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.198.126" port: 40351 } has no permanent_uuid. Determining permanent_uuid...
15:05:46.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.729398 13236 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } has no permanent_uuid. Determining permanent_uuid...
15:05:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.743701 13236 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } has no permanent_uuid. Determining permanent_uuid...
15:05:46.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:46.746111 13186 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.198.124:42251: connect: Connection refused (error 111)
15:05:46.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:46.750257 13236 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.198.124:42251: Network error: Client connection negotiation failed: client connection to 127.12.198.124:42251: connect: Connection refused (error 111)
15:05:46.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.799705 13236 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } attempt: 1
15:05:46.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:46.803606 13236 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.198.124:42251: Network error: Client connection negotiation failed: client connection to 127.12.198.124:42251: connect: Connection refused (error 111)
15:05:46.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.871964 13236 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } attempt: 2
15:05:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:46.875721 13236 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.198.124:42251: Network error: Client connection negotiation failed: client connection to 127.12.198.124:42251: connect: Connection refused (error 111)
15:05:46.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:46.967172 13236 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } attempt: 3
15:05:46.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:46.970896 13236 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.198.124:42251: Network error: Client connection negotiation failed: client connection to 127.12.198.124:42251: connect: Connection refused (error 111)
15:05:47.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:47.010666 13238 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:47.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:47.011207 13238 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:47.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:47.011850 13238 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:47.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:47.044667 13238 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:05:47.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:47.044953 13238 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:47.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:47.045264 13238 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:05:47.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:47.045481 13238 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:05:47.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:47.045672 13238 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:05:47.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:47.045816 13238 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38759/jwks.json
15:05:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:47.079953 13238 master_runner.cc:386] Master server non-default flags:
15:05:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.198.84:32891
15:05:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-2/data
15:05:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-2/wal
15:05:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.198.126:40351,127.12.198.125:37065,127.12.198.124:42251
15:05:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.198.124:42251
15:05:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38759/jwks.json
15:05:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-2/data/info.pb
15:05:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.198.124
15:05:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-2/logs
15:05:47.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:47.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:47.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:47.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:05:47.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:47.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:47.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:47.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 14:43:22 UTC on a633016ff5c9
15:05:47.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4112
15:05:47.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:47.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:47.081389 13238 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:47.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:47.083532 13238 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:47.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:47.099021 13249 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:47.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:47.114233 13236 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } attempt: 4
15:05:47.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:47.118058 13236 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.198.124:42251: Network error: Client connection negotiation failed: client connection to 127.12.198.124:42251: connect: Connection refused (error 111)
15:05:47.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:47.242595 13161 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } attempt: 7
15:05:47.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:47.255862 13161 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } has no permanent_uuid. Determining permanent_uuid...
15:05:47.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:47.262239 13161 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.198.124:42251: Network error: Client connection negotiation failed: client connection to 127.12.198.124:42251: connect: Connection refused (error 111)
15:05:47.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:47.320695 13161 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } attempt: 1
15:05:47.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:47.325629 13161 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.198.124:42251: Network error: Client connection negotiation failed: client connection to 127.12.198.124:42251: connect: Connection refused (error 111)
15:05:47.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:47.406997 13161 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } attempt: 2
15:05:47.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:47.411973 13161 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.198.124:42251: Network error: Client connection negotiation failed: client connection to 127.12.198.124:42251: connect: Connection refused (error 111)
15:05:47.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:47.417665 13236 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } attempt: 5
15:05:47.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:47.421850 13236 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.198.124:42251: Network error: Client connection negotiation failed: client connection to 127.12.198.124:42251: connect: Connection refused (error 111)
15:05:47.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:47.497354 13161 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } attempt: 3
15:05:47.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:47.501827 13161 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.198.124:42251: Network error: Client connection negotiation failed: client connection to 127.12.198.124:42251: connect: Connection refused (error 111)
15:05:47.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:47.642189 13161 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } attempt: 4
15:05:47.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:47.646646 13161 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.198.124:42251: Network error: Client connection negotiation failed: client connection to 127.12.198.124:42251: connect: Connection refused (error 111)
15:05:47.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:47.930011 13161 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } attempt: 5
15:05:47.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:47.934437 13161 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.198.124:42251: Network error: Client connection negotiation failed: client connection to 127.12.198.124:42251: connect: Connection refused (error 111)
15:05:47.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:47.969447 13236 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } attempt: 6
15:05:47.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:47.976455 13236 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.198.124:42251: Network error: Client connection negotiation failed: client connection to 127.12.198.124:42251: connect: Connection refused (error 111)
15:05:48.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:47.099071 13248 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:48.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:47.101423 13251 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:48.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:48.295571 13250 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
15:05:48.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.295620 13238 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:05:48.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.300580 13238 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:48.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.303298 13238 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:48.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.304656 13238 hybrid_clock.cc:648] HybridClock initialized: now 1736521548304613 us; error 52 us; skew 500 ppm
15:05:48.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.305508 13238 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:48.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.311793 13238 webserver.cc:458] Webserver started at http://127.12.198.124:40597/ using document root <none> and password file <none>
15:05:48.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.312717 13238 fs_manager.cc:362] Metadata directory not provided
15:05:48.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.312940 13238 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:48.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.313366 13238 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.318334 13238 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-2/data/instance:
15:05:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22e80c88384f41a9b9564d0b2555acbb"
15:05:48.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:05:48 on dist-test-slave-6prw"
15:05:48.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.319341 13238 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-2/wal/instance:
15:05:48.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22e80c88384f41a9b9564d0b2555acbb"
15:05:48.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:05:48 on dist-test-slave-6prw"
15:05:48.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.326681 13238 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
15:05:48.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.332033 13263 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:48.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.333158 13238 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
15:05:48.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.333429 13238 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-2/data,/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-2/wal
15:05:48.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22e80c88384f41a9b9564d0b2555acbb"
15:05:48.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:05:48 on dist-test-slave-6prw"
15:05:48.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.333720 13238 fs_report.cc:389] FS layout report
15:05:48.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:48.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-2/wal
15:05:48.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-2/wal
15:05:48.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-2/data/data
15:05:48.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:48.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:48.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:48.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:48.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:48.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:48.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:48.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:48.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:48.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:48.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:48.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.397656 13238 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.399052 13238 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.399461 13238 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.468791 13238 rpc_server.cc:307] RPC server started. Bound to: 127.12.198.124:42251
15:05:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.468883 13314 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.198.124:42251 every 8 connection(s)
15:05:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.471457 13238 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-2/data/info.pb
15:05:48.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.476691 13315 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:05:48.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.480885 13081 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu as pid 13238
15:05:48.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.481343 13081 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/master-2/wal/instance
15:05:48.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.487206 13161 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } attempt: 6
15:05:48.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.491941 13315 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.198.126" port: 40351 } has no permanent_uuid. Determining permanent_uuid...
15:05:48.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.525990 13315 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } has no permanent_uuid. Determining permanent_uuid...
15:05:48.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.534694 13161 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8: Bootstrap starting.
15:05:48.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.536631 13315 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } has no permanent_uuid. Determining permanent_uuid...
15:05:48.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.543821 13161 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8: Neither blocks nor log segments found. Creating new log.
15:05:48.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.546319 13161 log.cc:826] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8: Log is configured to *not* fsync() on all Append() calls
15:05:48.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.551364 13161 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8: No bootstrap required, opened a new log
15:05:48.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.554833 13315 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb: Bootstrap starting.
15:05:48.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.562587 13315 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb: Neither blocks nor log segments found. Creating new log.
15:05:48.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.565135 13315 log.cc:826] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb: Log is configured to *not* fsync() on all Append() calls
15:05:48.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.570912 13315 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb: No bootstrap required, opened a new log
15:05:48.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.577173 13161 raft_consensus.cc:357] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "840a94c2474c4985848a471061fd1ba8" member_type: VOTER last_known_addr { host: "127.12.198.126" port: 40351 } } peers { permanent_uuid: "86515feaf4ad4225bc17008d28478488" member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } } peers { permanent_uuid: "22e80c88384f41a9b9564d0b2555acbb" member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } }
15:05:48.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.578315 13161 raft_consensus.cc:383] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:05:48.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.578860 13161 raft_consensus.cc:738] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 840a94c2474c4985848a471061fd1ba8, State: Initialized, Role: FOLLOWER
15:05:48.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.580017 13161 consensus_queue.cc:260] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "840a94c2474c4985848a471061fd1ba8" member_type: VOTER last_known_addr { host: "127.12.198.126" port: 40351 } } peers { permanent_uuid: "86515feaf4ad4225bc17008d28478488" member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } } peers { permanent_uuid: "22e80c88384f41a9b9564d0b2555acbb" member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } }
15:05:48.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.582949 13327 sys_catalog.cc:455] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "840a94c2474c4985848a471061fd1ba8" member_type: VOTER last_known_addr { host: "127.12.198.126" port: 40351 } } peers { permanent_uuid: "86515feaf4ad4225bc17008d28478488" member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } } peers { permanent_uuid: "22e80c88384f41a9b9564d0b2555acbb" member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } } }
15:05:48.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.583796 13327 sys_catalog.cc:458] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [sys.catalog]: This master's current role is: FOLLOWER
15:05:48.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.584369 13161 sys_catalog.cc:564] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [sys.catalog]: configured and running, proceeding with master startup.
15:05:48.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:48.587426 13115 tablet.cc:2367] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:05:48.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.592049 13327 raft_consensus.cc:491] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:05:48.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.592614 13327 raft_consensus.cc:513] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "840a94c2474c4985848a471061fd1ba8" member_type: VOTER last_known_addr { host: "127.12.198.126" port: 40351 } } peers { permanent_uuid: "86515feaf4ad4225bc17008d28478488" member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } } peers { permanent_uuid: "22e80c88384f41a9b9564d0b2555acbb" member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } }
15:05:48.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.595219 13327 leader_election.cc:290] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 86515feaf4ad4225bc17008d28478488 (127.12.198.125:37065), 22e80c88384f41a9b9564d0b2555acbb (127.12.198.124:42251)
15:05:48.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.597913 13315 raft_consensus.cc:357] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "840a94c2474c4985848a471061fd1ba8" member_type: VOTER last_known_addr { host: "127.12.198.126" port: 40351 } } peers { permanent_uuid: "86515feaf4ad4225bc17008d28478488" member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } } peers { permanent_uuid: "22e80c88384f41a9b9564d0b2555acbb" member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } }
15:05:48.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.598783 13315 raft_consensus.cc:383] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:05:48.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.599143 13315 raft_consensus.cc:738] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 22e80c88384f41a9b9564d0b2555acbb, State: Initialized, Role: FOLLOWER
15:05:48.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.600032 13315 consensus_queue.cc:260] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "840a94c2474c4985848a471061fd1ba8" member_type: VOTER last_known_addr { host: "127.12.198.126" port: 40351 } } peers { permanent_uuid: "86515feaf4ad4225bc17008d28478488" member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } } peers { permanent_uuid: "22e80c88384f41a9b9564d0b2555acbb" member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } }
15:05:48.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.603349 13335 sys_catalog.cc:455] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "840a94c2474c4985848a471061fd1ba8" member_type: VOTER last_known_addr { host: "127.12.198.126" port: 40351 } } peers { permanent_uuid: "86515feaf4ad4225bc17008d28478488" member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } } peers { permanent_uuid: "22e80c88384f41a9b9564d0b2555acbb" member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } } }
15:05:48.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.604278 13335 sys_catalog.cc:458] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb [sys.catalog]: This master's current role is: FOLLOWER
15:05:48.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.605317 13315 sys_catalog.cc:564] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb [sys.catalog]: configured and running, proceeding with master startup.
15:05:48.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.605525 13290 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "840a94c2474c4985848a471061fd1ba8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "22e80c88384f41a9b9564d0b2555acbb" is_pre_election: true
15:05:48.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.606389 13290 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 840a94c2474c4985848a471061fd1ba8 in term 0.
15:05:48.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.607738 13111 leader_election.cc:304] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 22e80c88384f41a9b9564d0b2555acbb, 840a94c2474c4985848a471061fd1ba8; no voters:
15:05:48.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.608857 13327 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:05:48.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.609299 13327 raft_consensus.cc:491] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:05:48.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.609719 13327 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [term 0 FOLLOWER]: Advancing to term 1
15:05:48.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.617771 13327 raft_consensus.cc:513] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "840a94c2474c4985848a471061fd1ba8" member_type: VOTER last_known_addr { host: "127.12.198.126" port: 40351 } } peers { permanent_uuid: "86515feaf4ad4225bc17008d28478488" member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } } peers { permanent_uuid: "22e80c88384f41a9b9564d0b2555acbb" member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } }
15:05:48.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.623880 13327 leader_election.cc:290] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [CANDIDATE]: Term 1 election: Requested vote from peers 86515feaf4ad4225bc17008d28478488 (127.12.198.125:37065), 22e80c88384f41a9b9564d0b2555acbb (127.12.198.124:42251)
15:05:48.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.624985 13290 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "840a94c2474c4985848a471061fd1ba8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "22e80c88384f41a9b9564d0b2555acbb"
15:05:48.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.625622 13290 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb [term 0 FOLLOWER]: Advancing to term 1
15:05:48.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.633435 13290 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 840a94c2474c4985848a471061fd1ba8 in term 1.
15:05:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.637459 13111 leader_election.cc:304] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 22e80c88384f41a9b9564d0b2555acbb, 840a94c2474c4985848a471061fd1ba8; no voters:
15:05:48.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.638945 13327 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [term 1 FOLLOWER]: Leader election won for term 1
15:05:48.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.643867 13327 raft_consensus.cc:695] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [term 1 LEADER]: Becoming Leader. State: Replica: 840a94c2474c4985848a471061fd1ba8, State: Running, Role: LEADER
15:05:48.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.644918 13327 consensus_queue.cc:237] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "840a94c2474c4985848a471061fd1ba8" member_type: VOTER last_known_addr { host: "127.12.198.126" port: 40351 } } peers { permanent_uuid: "86515feaf4ad4225bc17008d28478488" member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } } peers { permanent_uuid: "22e80c88384f41a9b9564d0b2555acbb" member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } }
15:05:48.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:48.654524 13269 tablet.cc:2367] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:05:48.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:48.655251 13348 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:05:48.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:48.655577 13348 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:05:48.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.663319 13351 sys_catalog.cc:455] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 840a94c2474c4985848a471061fd1ba8. Latest consensus state: current_term: 1 leader_uuid: "840a94c2474c4985848a471061fd1ba8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "840a94c2474c4985848a471061fd1ba8" member_type: VOTER last_known_addr { host: "127.12.198.126" port: 40351 } } peers { permanent_uuid: "86515feaf4ad4225bc17008d28478488" member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } } peers { permanent_uuid: "22e80c88384f41a9b9564d0b2555acbb" member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } } }
15:05:48.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.664202 13351 sys_catalog.cc:458] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [sys.catalog]: This master's current role is: LEADER
15:05:48.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.642271 13093 master_runner.cc:186] Error getting master registration for 127.12.198.125:37065: OK, instance_id { permanent_uuid: "86515feaf4ad4225bc17008d28478488" instance_seqno: 1736521546666967 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:05:48.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.687366 13355 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
15:05:48.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.687932 13093 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:05:48.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:48.689307 13352 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:05:48.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:48.689841 13352 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:05:48.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.694960 13355 catalog_manager.cc:1485] Initializing Kudu cluster ID...
15:05:48.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.719524 13290 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb [term 1 FOLLOWER]: Refusing update from remote peer 840a94c2474c4985848a471061fd1ba8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:05:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.721604 13351 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "22e80c88384f41a9b9564d0b2555acbb" member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:05:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.675983 13238 master_runner.cc:186] Error getting master registration for 127.12.198.125:37065: OK, instance_id { permanent_uuid: "86515feaf4ad4225bc17008d28478488" instance_seqno: 1736521546666967 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:05:48.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.728974 13238 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:05:48.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.750599 13335 sys_catalog.cc:455] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb [sys.catalog]: SysCatalogTable state changed. Reason: New leader 840a94c2474c4985848a471061fd1ba8. Latest consensus state: current_term: 1 leader_uuid: "840a94c2474c4985848a471061fd1ba8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "840a94c2474c4985848a471061fd1ba8" member_type: VOTER last_known_addr { host: "127.12.198.126" port: 40351 } } peers { permanent_uuid: "86515feaf4ad4225bc17008d28478488" member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } } peers { permanent_uuid: "22e80c88384f41a9b9564d0b2555acbb" member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } } }
15:05:48.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.751351 13335 sys_catalog.cc:458] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb [sys.catalog]: This master's current role is: FOLLOWER
15:05:48.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.763720 13356 mvcc.cc:204] Tried to move back new op lower bound from 7112792263535763456 to 7112792263258943488. Current Snapshot: MvccSnapshot[applied={T|T < 7112792263535763456}]
15:05:48.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.777230 13335 sys_catalog.cc:455] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "840a94c2474c4985848a471061fd1ba8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "840a94c2474c4985848a471061fd1ba8" member_type: VOTER last_known_addr { host: "127.12.198.126" port: 40351 } } peers { permanent_uuid: "86515feaf4ad4225bc17008d28478488" member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } } peers { permanent_uuid: "22e80c88384f41a9b9564d0b2555acbb" member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } } }
15:05:48.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.777933 13335 sys_catalog.cc:458] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb [sys.catalog]: This master's current role is: FOLLOWER
15:05:48.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.785908 13355 catalog_manager.cc:1348] Generated new cluster ID: 07e16974458b43efa856f9a9b6ef3277
15:05:48.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.786262 13355 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
15:05:48.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.785902 13327 sys_catalog.cc:455] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "840a94c2474c4985848a471061fd1ba8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "840a94c2474c4985848a471061fd1ba8" member_type: VOTER last_known_addr { host: "127.12.198.126" port: 40351 } } peers { permanent_uuid: "86515feaf4ad4225bc17008d28478488" member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } } peers { permanent_uuid: "22e80c88384f41a9b9564d0b2555acbb" member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } } }
15:05:48.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.789074 13327 sys_catalog.cc:458] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [sys.catalog]: This master's current role is: LEADER
15:05:48.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.831437 13355 catalog_manager.cc:1371] Generated new certificate authority record
15:05:48.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.836410 13355 catalog_manager.cc:1505] Loading token signing keys...
15:05:48.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.868039 13355 catalog_manager.cc:5899] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8: Generated new TSK 0
15:05:48.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.869189 13355 catalog_manager.cc:1515] Initializing in-progress tserver states...
15:05:48.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:48.886576 13081 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:05:48.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:05:48.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-0/wal
15:05:48.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-0/data
15:05:48.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:48.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:48.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:48.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:48.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:48.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:48.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-0/logs
15:05:48.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-0/data/info.pb
15:05:48.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:48.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:48.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:48.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:48.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:48.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:48.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:48.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:48.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:05:48.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:48.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.198.65:0
15:05:48.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.198.65
15:05:48.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.198.65
15:05:48.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:48.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.198.126:40351,127.12.198.125:37065,127.12.198.124:42251
15:05:48.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.198.84:32891
15:05:48.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:48.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:48.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:05:49.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.037001 13236 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } attempt: 7
15:05:49.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.084830 13236 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 86515feaf4ad4225bc17008d28478488: Bootstrap starting.
15:05:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.092963 13236 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 86515feaf4ad4225bc17008d28478488: Neither blocks nor log segments found. Creating new log.
15:05:49.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.105566 13236 log.cc:826] T 00000000000000000000000000000000 P 86515feaf4ad4225bc17008d28478488: Log is configured to *not* fsync() on all Append() calls
15:05:49.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.128952 13236 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 86515feaf4ad4225bc17008d28478488: No bootstrap required, opened a new log
15:05:49.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.172525 13236 raft_consensus.cc:357] T 00000000000000000000000000000000 P 86515feaf4ad4225bc17008d28478488 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "840a94c2474c4985848a471061fd1ba8" member_type: VOTER last_known_addr { host: "127.12.198.126" port: 40351 } } peers { permanent_uuid: "86515feaf4ad4225bc17008d28478488" member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } } peers { permanent_uuid: "22e80c88384f41a9b9564d0b2555acbb" member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } }
15:05:49.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.173821 13236 raft_consensus.cc:383] T 00000000000000000000000000000000 P 86515feaf4ad4225bc17008d28478488 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:05:49.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.174338 13236 raft_consensus.cc:738] T 00000000000000000000000000000000 P 86515feaf4ad4225bc17008d28478488 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 86515feaf4ad4225bc17008d28478488, State: Initialized, Role: FOLLOWER
15:05:49.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.175683 13236 consensus_queue.cc:260] T 00000000000000000000000000000000 P 86515feaf4ad4225bc17008d28478488 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "840a94c2474c4985848a471061fd1ba8" member_type: VOTER last_known_addr { host: "127.12.198.126" port: 40351 } } peers { permanent_uuid: "86515feaf4ad4225bc17008d28478488" member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } } peers { permanent_uuid: "22e80c88384f41a9b9564d0b2555acbb" member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } }
15:05:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.189672 13364 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86515feaf4ad4225bc17008d28478488 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "840a94c2474c4985848a471061fd1ba8" member_type: VOTER last_known_addr { host: "127.12.198.126" port: 40351 } } peers { permanent_uuid: "86515feaf4ad4225bc17008d28478488" member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } } peers { permanent_uuid: "22e80c88384f41a9b9564d0b2555acbb" member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } } }
15:05:49.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.192443 13364 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86515feaf4ad4225bc17008d28478488 [sys.catalog]: This master's current role is: FOLLOWER
15:05:49.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.194770 13210 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "840a94c2474c4985848a471061fd1ba8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "86515feaf4ad4225bc17008d28478488"
15:05:49.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.195154 13209 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 86515feaf4ad4225bc17008d28478488 [term 0 FOLLOWER]: Advancing to term 1
15:05:49.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.196453 13211 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "840a94c2474c4985848a471061fd1ba8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "86515feaf4ad4225bc17008d28478488" is_pre_election: true
15:05:49.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.198899 13236 sys_catalog.cc:564] T 00000000000000000000000000000000 P 86515feaf4ad4225bc17008d28478488 [sys.catalog]: configured and running, proceeding with master startup.
15:05:49.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.201905 13209 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 86515feaf4ad4225bc17008d28478488 [term 1 FOLLOWER]: Refusing update from remote peer 840a94c2474c4985848a471061fd1ba8: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:05:49.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.203649 13327 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "86515feaf4ad4225bc17008d28478488" member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:05:49.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.289706 13364 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86515feaf4ad4225bc17008d28478488 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 840a94c2474c4985848a471061fd1ba8. Latest consensus state: current_term: 1 leader_uuid: "840a94c2474c4985848a471061fd1ba8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "840a94c2474c4985848a471061fd1ba8" member_type: VOTER last_known_addr { host: "127.12.198.126" port: 40351 } } peers { permanent_uuid: "86515feaf4ad4225bc17008d28478488" member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } } peers { permanent_uuid: "22e80c88384f41a9b9564d0b2555acbb" member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } } }
15:05:49.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.289829 13369 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86515feaf4ad4225bc17008d28478488 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "840a94c2474c4985848a471061fd1ba8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "840a94c2474c4985848a471061fd1ba8" member_type: VOTER last_known_addr { host: "127.12.198.126" port: 40351 } } peers { permanent_uuid: "86515feaf4ad4225bc17008d28478488" member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } } peers { permanent_uuid: "22e80c88384f41a9b9564d0b2555acbb" member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } } }
15:05:49.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.290553 13364 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86515feaf4ad4225bc17008d28478488 [sys.catalog]: This master's current role is: FOLLOWER
15:05:49.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.290565 13369 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86515feaf4ad4225bc17008d28478488 [sys.catalog]: This master's current role is: FOLLOWER
15:05:49.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.325520 13380 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:05:49.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:49.493577 13361 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:49.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:49.494225 13361 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:49.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:49.494998 13361 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:49.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:49.551378 13361 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:49.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:49.552739 13361 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.198.65
15:05:49.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.613803 13361 tablet_server_runner.cc:78] Tablet server non-default flags:
15:05:49.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:49.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.198.84:32891
15:05:49.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:49.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:49.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-0/data
15:05:49.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-0/wal
15:05:49.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:49.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:49.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.198.65:0
15:05:49.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:49.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:49.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:49.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-0/data/info.pb
15:05:49.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.198.65
15:05:49.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.198.126:40351,127.12.198.125:37065,127.12.198.124:42251
15:05:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.198.65
15:05:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-0/logs
15:05:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:05:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 14:43:22 UTC on a633016ff5c9
15:05:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4112
15:05:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:49.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.615535 13361 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:49.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.617589 13361 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:49.637162 13386 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:49.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.695948 13352 catalog_manager.cc:1260] Loaded cluster ID: 07e16974458b43efa856f9a9b6ef3277
15:05:49.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.696229 13352 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb: loading cluster ID for follower catalog manager: success
15:05:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.703521 13352 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb: acquiring CA information for follower catalog manager: success
15:05:49.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.729492 13352 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 22e80c88384f41a9b9564d0b2555acbb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:05:49.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.773785 13327 sys_catalog.cc:455] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "840a94c2474c4985848a471061fd1ba8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "840a94c2474c4985848a471061fd1ba8" member_type: VOTER last_known_addr { host: "127.12.198.126" port: 40351 } } peers { permanent_uuid: "86515feaf4ad4225bc17008d28478488" member_type: VOTER last_known_addr { host: "127.12.198.125" port: 37065 } } peers { permanent_uuid: "22e80c88384f41a9b9564d0b2555acbb" member_type: VOTER last_known_addr { host: "127.12.198.124" port: 42251 } } }
15:05:49.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.774874 13327 sys_catalog.cc:458] T 00000000000000000000000000000000 P 840a94c2474c4985848a471061fd1ba8 [sys.catalog]: This master's current role is: LEADER
15:05:50.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:50.352666 13380 catalog_manager.cc:1260] Loaded cluster ID: 07e16974458b43efa856f9a9b6ef3277
15:05:50.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:50.353278 13380 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 86515feaf4ad4225bc17008d28478488: loading cluster ID for follower catalog manager: success
15:05:50.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:50.360616 13380 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 86515feaf4ad4225bc17008d28478488: acquiring CA information for follower catalog manager: success
15:05:50.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:50.377789 13380 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 86515feaf4ad4225bc17008d28478488: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:05:51.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:49.644945 13387 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:51.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:49.645915 13361 server_base.cc:1034] running on GCE node
15:05:51.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:49.646600 13389 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:51.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.146302 13361 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:51.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.157378 13361 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:51.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.158830 13361 hybrid_clock.cc:648] HybridClock initialized: now 1736521551158787 us; error 53 us; skew 500 ppm
15:05:51.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.159991 13361 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:51.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.197150 13361 webserver.cc:458] Webserver started at http://127.12.198.65:43829/ using document root <none> and password file <none>
15:05:51.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.198731 13361 fs_manager.cc:362] Metadata directory not provided
15:05:51.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.199239 13361 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:51.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.199963 13361 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:51.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.208096 13361 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-0/data/instance:
15:05:51.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dbc285696a7d41bdafd545d8b0474c06"
15:05:51.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:05:51 on dist-test-slave-6prw"
15:05:51.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.211243 13361 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-0/wal/instance:
15:05:51.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dbc285696a7d41bdafd545d8b0474c06"
15:05:51.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:05:51 on dist-test-slave-6prw"
15:05:51.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.227370 13361 fs_manager.cc:696] Time spent creating directory manager: real 0.015s user 0.003s sys 0.009s
15:05:51.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.242275 13400 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:51.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.243880 13361 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.001s sys 0.007s
15:05:51.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.244540 13361 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-0/data,/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-0/wal
15:05:51.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dbc285696a7d41bdafd545d8b0474c06"
15:05:51.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:05:51 on dist-test-slave-6prw"
15:05:51.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.245250 13361 fs_report.cc:389] FS layout report
15:05:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-0/wal
15:05:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-0/wal
15:05:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-0/data/data
15:05:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:51.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.356989 13361 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:51.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.359311 13361 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:51.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.360121 13361 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:51.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.369824 13361 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:05:51.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.383417 13361 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:05:51.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.383986 13361 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.002s user 0.001s sys 0.000s
15:05:51.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.384470 13361 ts_tablet_manager.cc:610] Registered 0 tablets
15:05:51.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.384804 13361 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:05:51.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.665237 13512 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.198.65:45057 every 8 connection(s)
15:05:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.665838 13361 rpc_server.cc:307] RPC server started. Bound to: 127.12.198.65:45057
15:05:51.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.668435 13361 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-0/data/info.pb
15:05:51.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.671005 13081 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu as pid 13361
15:05:51.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.671447 13081 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-0/wal/instance
15:05:51.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.682729 13081 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:05:51.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:05:51.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-1/wal
15:05:51.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-1/data
15:05:51.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:51.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:51.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:51.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:51.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:51.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:51.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-1/logs
15:05:51.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-1/data/info.pb
15:05:51.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:51.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:51.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:51.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:51.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:51.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:51.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:51.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:51.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:05:51.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:51.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.198.66:0
15:05:51.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.198.66
15:05:51.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.198.66
15:05:51.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:51.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.198.126:40351,127.12.198.125:37065,127.12.198.124:42251
15:05:51.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.198.84:32891
15:05:51.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:51.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:51.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:05:51.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.751619 13513 heartbeater.cc:346] Connected to a master server at 127.12.198.125:37065
15:05:51.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.752434 13513 heartbeater.cc:463] Registering TS with master...
15:05:51.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.753579 13513 heartbeater.cc:510] Master 127.12.198.125:37065 requested a full tablet report, sending...
15:05:51.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.757689 13201 ts_manager.cc:194] Registered new tserver with Master: dbc285696a7d41bdafd545d8b0474c06 (127.12.198.65:45057)
15:05:51.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.771116 13514 heartbeater.cc:346] Connected to a master server at 127.12.198.124:42251
15:05:51.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.771499 13514 heartbeater.cc:463] Registering TS with master...
15:05:51.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.772421 13514 heartbeater.cc:510] Master 127.12.198.124:42251 requested a full tablet report, sending...
15:05:51.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.773255 13518 heartbeater.cc:346] Connected to a master server at 127.12.198.126:40351
15:05:51.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.773597 13518 heartbeater.cc:463] Registering TS with master...
15:05:51.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.774449 13518 heartbeater.cc:510] Master 127.12.198.126:40351 requested a full tablet report, sending...
15:05:51.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.776731 13126 ts_manager.cc:194] Registered new tserver with Master: dbc285696a7d41bdafd545d8b0474c06 (127.12.198.65:45057)
15:05:51.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.777230 13280 ts_manager.cc:194] Registered new tserver with Master: dbc285696a7d41bdafd545d8b0474c06 (127.12.198.65:45057)
15:05:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:51.779379 13126 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.12.198.65:39471
15:05:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:52.077136 13517 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:52.077607 13517 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:52.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:52.078040 13517 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:52.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:52.109468 13517 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:52.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:52.110288 13517 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.198.66
15:05:52.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:52.144955 13517 tablet_server_runner.cc:78] Tablet server non-default flags:
15:05:52.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:52.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.198.84:32891
15:05:52.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:52.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:52.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-1/data
15:05:52.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-1/wal
15:05:52.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:52.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:52.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.198.66:0
15:05:52.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:52.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:52.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:52.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-1/data/info.pb
15:05:52.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.198.66
15:05:52.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:52.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.198.126:40351,127.12.198.125:37065,127.12.198.124:42251
15:05:52.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:52.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:52.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:52.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:52.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:52.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.198.66
15:05:52.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-1/logs
15:05:52.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:52.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:52.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:52.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:05:52.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:52.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:52.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:52.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 14:43:22 UTC on a633016ff5c9
15:05:52.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4112
15:05:52.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:52.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:52.146687 13517 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:52.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:52.148767 13517 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:52.170331 13532 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:52.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:52.785174 13518 heartbeater.cc:502] Master 127.12.198.126:40351 was elected leader, sending a full tablet report...
15:05:53.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:52.171147 13529 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:53.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:52.171814 13530 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:53.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:52.172664 13517 server_base.cc:1034] running on GCE node
15:05:53.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.378058 13517 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.380297 13517 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.381733 13517 hybrid_clock.cc:648] HybridClock initialized: now 1736521553381706 us; error 34 us; skew 500 ppm
15:05:53.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.382593 13517 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:53.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.390020 13517 webserver.cc:458] Webserver started at http://127.12.198.66:46139/ using document root <none> and password file <none>
15:05:53.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.390949 13517 fs_manager.cc:362] Metadata directory not provided
15:05:53.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.391170 13517 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:53.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.391654 13517 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:53.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.396597 13517 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-1/data/instance:
15:05:53.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3b194efa8f7845998e82d16a184b9c52"
15:05:53.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:05:53 on dist-test-slave-6prw"
15:05:53.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.397845 13517 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-1/wal/instance:
15:05:53.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3b194efa8f7845998e82d16a184b9c52"
15:05:53.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:05:53 on dist-test-slave-6prw"
15:05:53.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.405021 13517 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
15:05:53.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.410844 13540 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:53.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.411811 13517 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
15:05:53.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.412161 13517 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-1/data,/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-1/wal
15:05:53.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3b194efa8f7845998e82d16a184b9c52"
15:05:53.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:05:53 on dist-test-slave-6prw"
15:05:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.412562 13517 fs_report.cc:389] FS layout report
15:05:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-1/wal
15:05:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-1/wal
15:05:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-1/data/data
15:05:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:53.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.472268 13517 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:53.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.474001 13517 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:53.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.474530 13517 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:53.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.477842 13517 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:05:53.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.482291 13517 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:05:53.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.482511 13517 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:05:53.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.482828 13517 ts_tablet_manager.cc:610] Registered 0 tablets
15:05:53.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.483019 13517 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:05:53.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.629511 13517 rpc_server.cc:307] RPC server started. Bound to: 127.12.198.66:36681
15:05:53.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.629608 13653 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.198.66:36681 every 8 connection(s)
15:05:53.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.631996 13517 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-1/data/info.pb
15:05:53.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.632550 13081 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu as pid 13517
15:05:53.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.632983 13081 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-1/wal/instance
15:05:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.644189 13081 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:05:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:05:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-2/wal
15:05:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-2/data
15:05:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-2/logs
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-2/data/info.pb
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.198.67:0
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.198.67
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.198.67
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.198.126:40351,127.12.198.125:37065,127.12.198.124:42251
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.198.84:32891
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:05:53.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.686762 13654 heartbeater.cc:346] Connected to a master server at 127.12.198.125:37065
15:05:53.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.687299 13654 heartbeater.cc:463] Registering TS with master...
15:05:53.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.689254 13654 heartbeater.cc:510] Master 127.12.198.125:37065 requested a full tablet report, sending...
15:05:53.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.692438 13655 heartbeater.cc:346] Connected to a master server at 127.12.198.124:42251
15:05:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.692970 13655 heartbeater.cc:463] Registering TS with master...
15:05:53.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.693181 13201 ts_manager.cc:194] Registered new tserver with Master: 3b194efa8f7845998e82d16a184b9c52 (127.12.198.66:36681)
15:05:53.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.694444 13655 heartbeater.cc:510] Master 127.12.198.124:42251 requested a full tablet report, sending...
15:05:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.702134 13280 ts_manager.cc:194] Registered new tserver with Master: 3b194efa8f7845998e82d16a184b9c52 (127.12.198.66:36681)
15:05:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.707881 13661 heartbeater.cc:346] Connected to a master server at 127.12.198.126:40351
15:05:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.708271 13661 heartbeater.cc:463] Registering TS with master...
15:05:53.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.709108 13661 heartbeater.cc:510] Master 127.12.198.126:40351 requested a full tablet report, sending...
15:05:53.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.711300 13126 ts_manager.cc:194] Registered new tserver with Master: 3b194efa8f7845998e82d16a184b9c52 (127.12.198.66:36681)
15:05:53.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:53.713768 13126 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.12.198.66:53487
15:05:54.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:54.005579 13660 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:54.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:54.006047 13660 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:54.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:54.006480 13660 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:54.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:54.037612 13660 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:54.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:54.038409 13660 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.198.67
15:05:54.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:54.074389 13660 tablet_server_runner.cc:78] Tablet server non-default flags:
15:05:54.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:54.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.198.84:32891
15:05:54.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:54.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:54.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-2/data
15:05:54.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-2/wal
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.198.67:0
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-2/data/info.pb
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.198.67
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.198.126:40351,127.12.198.125:37065,127.12.198.124:42251
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.198.67
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-2/logs
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 14:43:22 UTC on a633016ff5c9
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4112
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:54.075652 13660 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:54.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:54.077229 13660 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:54.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:54.095489 13671 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:54.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:54.717232 13661 heartbeater.cc:502] Master 127.12.198.126:40351 was elected leader, sending a full tablet report...
15:05:55.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:54.096292 13673 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:55.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:54.096733 13670 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:55.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:54.097467 13660 server_base.cc:1034] running on GCE node
15:05:55.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.275712 13660 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:55.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.277936 13660 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:55.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.279271 13660 hybrid_clock.cc:648] HybridClock initialized: now 1736521555279237 us; error 58 us; skew 500 ppm
15:05:55.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.280011 13660 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:55.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.286250 13660 webserver.cc:458] Webserver started at http://127.12.198.67:43719/ using document root <none> and password file <none>
15:05:55.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.287083 13660 fs_manager.cc:362] Metadata directory not provided
15:05:55.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.287267 13660 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:55.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.287688 13660 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:55.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.292433 13660 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-2/data/instance:
15:05:55.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "323073dbeb8f4bc5a59a9372461cb6bf"
15:05:55.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:05:55 on dist-test-slave-6prw"
15:05:55.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.293665 13660 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-2/wal/instance:
15:05:55.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "323073dbeb8f4bc5a59a9372461cb6bf"
15:05:55.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:05:55 on dist-test-slave-6prw"
15:05:55.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.300808 13660 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
15:05:55.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.306098 13681 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:55.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.307027 13660 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.002s
15:05:55.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.307360 13660 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-2/data,/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-2/wal
15:05:55.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "323073dbeb8f4bc5a59a9372461cb6bf"
15:05:55.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:05:55 on dist-test-slave-6prw"
15:05:55.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.307687 13660 fs_report.cc:389] FS layout report
15:05:55.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:55.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-2/wal
15:05:55.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-2/wal
15:05:55.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-2/data/data
15:05:55.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:55.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:55.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:55.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:55.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:55.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:55.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:55.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:55.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:55.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:55.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:55.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.365705 13660 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:55.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.367180 13660 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:55.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.367652 13660 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:55.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.370304 13660 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:05:55.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.374243 13660 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:05:55.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.374446 13660 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:05:55.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.374755 13660 ts_tablet_manager.cc:610] Registered 0 tablets
15:05:55.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.374926 13660 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:05:55.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.524483 13660 rpc_server.cc:307] RPC server started. Bound to: 127.12.198.67:44337
15:05:55.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.524596 13793 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.198.67:44337 every 8 connection(s)
15:05:55.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.526968 13660 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-2/data/info.pb
15:05:55.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.532963 13081 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu as pid 13660
15:05:55.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.533478 13081 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster7341444615076556258/ts-2/wal/instance
15:05:55.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.568655 13794 heartbeater.cc:346] Connected to a master server at 127.12.198.125:37065
15:05:55.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.569126 13794 heartbeater.cc:463] Registering TS with master...
15:05:55.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.569856 13795 heartbeater.cc:346] Connected to a master server at 127.12.198.124:42251
15:05:55.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.570194 13795 heartbeater.cc:463] Registering TS with master...
15:05:55.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.570369 13794 heartbeater.cc:510] Master 127.12.198.125:37065 requested a full tablet report, sending...
15:05:55.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.571190 13795 heartbeater.cc:510] Master 127.12.198.124:42251 requested a full tablet report, sending...
15:05:55.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.571503 13796 heartbeater.cc:346] Connected to a master server at 127.12.198.126:40351
15:05:55.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.571826 13796 heartbeater.cc:463] Registering TS with master...
15:05:55.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.572536 13796 heartbeater.cc:510] Master 127.12.198.126:40351 requested a full tablet report, sending...
15:05:55.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.573872 13201 ts_manager.cc:194] Registered new tserver with Master: 323073dbeb8f4bc5a59a9372461cb6bf (127.12.198.67:44337)
15:05:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.574623 13126 ts_manager.cc:194] Registered new tserver with Master: 323073dbeb8f4bc5a59a9372461cb6bf (127.12.198.67:44337)
15:05:55.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.574685 13279 ts_manager.cc:194] Registered new tserver with Master: 323073dbeb8f4bc5a59a9372461cb6bf (127.12.198.67:44337)
15:05:55.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.575989 13126 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.12.198.67:34645
15:05:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:55.581408 13081 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
15:05:55.592 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:05:55.593 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
15:05:55.603 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.12.198.126"
port: 40351
}
bound_http_address {
host: "127.12.198.126"
port: 39227
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.12.198.125"
port: 37065
}
bound_http_address {
host: "127.12.198.125"
port: 33873
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.12.198.124"
port: 42251
}
bound_http_address {
host: "127.12.198.124"
port: 40597
}
}
}
15:05:55.607 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:05:55.609 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.12.198.65"
port: 45057
}
bound_http_address {
host: "127.12.198.65"
port: 43829
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.12.198.66"
port: 36681
}
bound_http_address {
host: "127.12.198.66"
port: 46139
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.12.198.67"
port: 44337
}
bound_http_address {
host: "127.12.198.67"
port: 43719
}
}
}
15:05:55.609 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
15:05:55.830 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
15:05:55.903 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskyZ3UZe/test-tmp
15:05:55.903 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:05:55.904 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskyZ3UZe/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:05:55.907 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
15:05:56.243 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:05:56.243 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:05:56.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:56.246363 13805 mini_oidc.cc:150] Starting JWKS server
15:05:56.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:56.250047 13805 webserver.cc:458] Webserver started at http://127.0.0.1:42691/ using document root <none> and password file <none>
15:05:56.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:56.250434 13805 mini_oidc.cc:186] Starting OIDC Discovery server
15:05:56.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:56.251994 13805 webserver.cc:458] Webserver started at http://127.0.0.1:43619/ using document root <none> and password file <none>
15:05:56.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
15:05:56.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/principal' for realm 'KRBTEST.COM',
15:05:56.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
15:05:56.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:05:56 dist-test-slave-6prw krb5kdc[13817](info): setting up network...
15:05:56.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
15:05:56.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:05:56 dist-test-slave-6prw krb5kdc[13817](info): set up 2 sockets
15:05:56.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:05:56 dist-test-slave-6prw krb5kdc[13817](info): commencing operation
15:05:56.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
15:05:56.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:56.579156 13796 heartbeater.cc:502] Master 127.12.198.126:40351 was elected leader, sending a full tablet report...
15:05:57.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:57.366571 13805 mini_kdc.cc:121] Time spent starting KDC: real 1.076s user 0.006s sys 0.013s
15:05:57.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
15:05:57.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:05:57.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
15:05:57.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
15:05:57.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:05:57.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
15:05:57.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
15:05:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:05:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
15:05:57.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:05:57.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/test-user.keytab.
15:05:57.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/test-user.keytab.
15:05:57.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:05:57 dist-test-slave-6prw krb5kdc[13817](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521557, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:05:57.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:05:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-10T15:05:57Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
15:05:57.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-10T15:05:57Z Disabled control of system clock
15:05:57.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.13.123.126@KRBTEST.COM; defaulting to no policy
15:05:57.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:05:57.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.13.123.126@KRBTEST.COM" created.
15:05:57.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:05:57.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.123.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.126.keytab.
15:05:57.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.123.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.126.keytab.
15:05:57.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:57.549482 13805 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/wal
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/data
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/logs
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/data/info.pb
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.123.126:46369
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.123.126
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.126.keytab
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.123.126
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.123.84:33031
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.123.126:46369,127.13.123.125:44729,127.13.123.124:43073
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42691/jwks.json
15:05:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:05:57.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:57.848168 13831 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:57.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:57.848809 13831 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:57.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:57.849505 13831 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:57.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:57.880209 13831 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:05:57.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:57.880496 13831 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:05:57.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:57.880882 13831 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:57.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:57.881202 13831 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:05:57.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:57.881481 13831 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:05:57.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:57.881753 13831 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:05:57.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:57.881960 13831 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42691/jwks.json
15:05:57.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:57.918948 13831 master_runner.cc:386] Master server non-default flags:
15:05:57.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:57.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.123.84:33031
15:05:57.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:57.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:57.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/data
15:05:57.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/wal
15:05:57.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:05:57.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:57.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.123.126:46369,127.13.123.125:44729,127.13.123.124:43073
15:05:57.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.126.keytab
15:05:57.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.123.126
15:05:57.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:57.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:57.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:57.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.123.126:46369
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42691/jwks.json
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/data/info.pb
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.123.126
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/logs
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 14:43:22 UTC on a633016ff5c9
15:05:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4112
15:05:57.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:57.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:57.920620 13831 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:57.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:57.921710 13831 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:57.937419 13837 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:59.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:57.938375 13838 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:59.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:57.939594 13840 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:05:59.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:57.940725 13831 server_base.cc:1034] running on GCE node
15:05:59.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.183030 13831 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:05:59.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.185595 13831 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:05:59.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.187034 13831 hybrid_clock.cc:648] HybridClock initialized: now 1736521559186987 us; error 76 us; skew 500 ppm
15:05:59.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.187850 13831 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:05:59.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:05:59 dist-test-slave-6prw krb5kdc[13817](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521559, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:05:59.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.193878 13831 init.cc:377] Logged in from keytab as kudu/127.13.123.126@KRBTEST.COM (short username kudu)
15:05:59.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.199493 13831 webserver.cc:458] Webserver started at http://127.13.123.126:38515/ using document root <none> and password file <none>
15:05:59.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.200491 13831 fs_manager.cc:362] Metadata directory not provided
15:05:59.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.200716 13831 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:05:59.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.201251 13831 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:05:59.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.206059 13831 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/data/instance:
15:05:59.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fbe595fcd2124f2b9c575098f0832994"
15:05:59.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:05:59 on dist-test-slave-6prw"
15:05:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.207254 13831 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/wal/instance:
15:05:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fbe595fcd2124f2b9c575098f0832994"
15:05:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:05:59 on dist-test-slave-6prw"
15:05:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.214728 13831 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
15:05:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.220475 13848 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:05:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.221485 13831 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
15:05:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.221767 13831 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/data,/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/wal
15:05:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fbe595fcd2124f2b9c575098f0832994"
15:05:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:05:59 on dist-test-slave-6prw"
15:05:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.222070 13831 fs_report.cc:389] FS layout report
15:05:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:05:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/wal
15:05:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/wal
15:05:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/data/data
15:05:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:05:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:05:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:05:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:05:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:05:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:05:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:05:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:05:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:05:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:05:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:05:59.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.269554 13831 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:05:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.274559 13831 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:05:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.274973 13831 kserver.cc:163] Server-wide thread pool size limit: 3276
15:05:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.351608 13831 rpc_server.cc:307] RPC server started. Bound to: 127.13.123.126:46369
15:05:59.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.351711 13899 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.123.126:46369 every 8 connection(s)
15:05:59.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.354197 13831 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/data/info.pb
15:05:59.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.360334 13900 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:05:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.362401 13805 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu as pid 13831
15:05:59.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.362946 13805 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/wal/instance
15:05:59.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.375887 13900 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } has no permanent_uuid. Determining permanent_uuid...
15:05:59.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.13.123.125@KRBTEST.COM; defaulting to no policy
15:05:59.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:05:59.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.13.123.125@KRBTEST.COM" created.
15:05:59.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:05:59 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521559, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.126@KRBTEST.COM for kudu/127.13.123.126@KRBTEST.COM
15:05:59.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:05:59.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.123.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.125.keytab.
15:05:59.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.123.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.125.keytab.
15:05:59.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.410405 13805 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:05:59.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:05:59.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/wal
15:05:59.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/data
15:05:59.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/logs
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/data/info.pb
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.123.125:44729
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.123.125
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.125.keytab
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.123.125
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.123.84:33031
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.123.126:46369,127.13.123.125:44729,127.13.123.124:43073
15:05:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42691/jwks.json
15:05:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:05:59.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.416509 13900 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } has no permanent_uuid. Determining permanent_uuid...
15:05:59.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:59.418521 13851 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.123.125:44729: connect: Connection refused (error 111)
15:05:59.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:59.423439 13900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.123.125:44729: Network error: Client connection negotiation failed: client connection to 127.13.123.125:44729: connect: Connection refused (error 111)
15:05:59.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.472890 13900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } attempt: 1
15:05:59.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:59.476410 13900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.123.125:44729: Network error: Client connection negotiation failed: client connection to 127.13.123.125:44729: connect: Connection refused (error 111)
15:05:59.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.544764 13900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } attempt: 2
15:05:59.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:59.548264 13900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.123.125:44729: Network error: Client connection negotiation failed: client connection to 127.13.123.125:44729: connect: Connection refused (error 111)
15:05:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.639695 13900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } attempt: 3
15:05:59.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:59.643751 13900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.123.125:44729: Network error: Client connection negotiation failed: client connection to 127.13.123.125:44729: connect: Connection refused (error 111)
15:05:59.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:59.736770 13906 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:05:59.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:59.737407 13906 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:05:59.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:59.738090 13906 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:05:59.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:59.770063 13906 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:05:59.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:59.770323 13906 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:05:59.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:59.770700 13906 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:05:59.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:59.770985 13906 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:05:59.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:59.771257 13906 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:05:59.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:59.771519 13906 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:05:59.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:59.771731 13906 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42691/jwks.json
15:05:59.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.787146 13900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } attempt: 4
15:05:59.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:59.790482 13900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.123.125:44729: Network error: Client connection negotiation failed: client connection to 127.13.123.125:44729: connect: Connection refused (error 111)
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.810614 13906 master_runner.cc:386] Master server non-default flags:
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.123.84:33031
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/data
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/wal
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.123.126:46369,127.13.123.125:44729,127.13.123.124:43073
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.125.keytab
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.123.125
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.123.125:44729
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42691/jwks.json
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/data/info.pb
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.123.125
15:05:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:05:59.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:05:59.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:05:59.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:05:59.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:05:59.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:05:59.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/logs
15:05:59.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:05:59.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:05:59.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:05:59.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:05:59.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:05:59.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:05:59.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:05:59.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 14:43:22 UTC on a633016ff5c9
15:05:59.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4112
15:05:59.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:05:59.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.812325 13906 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:05:59.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.813370 13906 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:05:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:59.830557 13914 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:00.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:00.089886 13900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } attempt: 5
15:06:00.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:00.094033 13900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.123.125:44729: Network error: Client connection negotiation failed: client connection to 127.13.123.125:44729: connect: Connection refused (error 111)
15:06:00.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:00.161473 13509 debug-util.cc:398] Leaking SignalData structure 0x7b08000b1e60 after lost signal to thread 13381
15:06:00.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:00.162770 13509 debug-util.cc:398] Leaking SignalData structure 0x7b08000bbc40 after lost signal to thread 13512
15:06:00.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:00.641376 13900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } attempt: 6
15:06:00.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:00.658114 13900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.123.125:44729: Network error: Client connection negotiation failed: client connection to 127.13.123.125:44729: connect: Connection refused (error 111)
15:06:01.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:59.831473 13915 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:01.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:05:59.832047 13906 server_base.cc:1034] running on GCE node
15:06:01.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:05:59.830922 13917 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:01.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.092621 13906 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:01.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.096079 13906 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:01.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.097607 13906 hybrid_clock.cc:648] HybridClock initialized: now 1736521561097527 us; error 86 us; skew 500 ppm
15:06:01.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.098816 13906 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:01.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:01 dist-test-slave-6prw krb5kdc[13817](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521561, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:01.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.107749 13906 init.cc:377] Logged in from keytab as kudu/127.13.123.125@KRBTEST.COM (short username kudu)
15:06:01.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.114363 13906 webserver.cc:458] Webserver started at http://127.13.123.125:35827/ using document root <none> and password file <none>
15:06:01.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.115743 13906 fs_manager.cc:362] Metadata directory not provided
15:06:01.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.116046 13906 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:01.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.116667 13906 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:06:01.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.124750 13906 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/data/instance:
15:06:01.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0"
15:06:01.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:06:01 on dist-test-slave-6prw"
15:06:01.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.126454 13906 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/wal/instance:
15:06:01.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0"
15:06:01.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:06:01 on dist-test-slave-6prw"
15:06:01.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.137424 13906 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.011s sys 0.000s
15:06:01.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.145295 13927 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:01.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.146564 13906 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
15:06:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.146878 13906 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/data,/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/wal
15:06:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0"
15:06:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:06:01 on dist-test-slave-6prw"
15:06:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.147215 13906 fs_report.cc:389] FS layout report
15:06:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/wal
15:06:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/wal
15:06:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/data/data
15:06:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:01.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:01.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:01.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:01.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:01.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:01.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:01.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.221778 13906 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:01.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.229408 13906 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:01.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.229832 13906 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.322687 13906 rpc_server.cc:307] RPC server started. Bound to: 127.13.123.125:44729
15:06:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.322803 13978 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.123.125:44729 every 8 connection(s)
15:06:01.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.325965 13906 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/data/info.pb
15:06:01.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.330233 13805 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu as pid 13906
15:06:01.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.330720 13805 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/wal/instance
15:06:01.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.334316 13979 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:06:01.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.13.123.124@KRBTEST.COM; defaulting to no policy
15:06:01.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:01.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.13.123.124@KRBTEST.COM" created.
15:06:01.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.357019 13979 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } has no permanent_uuid. Determining permanent_uuid...
15:06:01.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:01.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.123.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.124.keytab.
15:06:01.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.123.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.124.keytab.
15:06:01.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:01 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521561, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.125@KRBTEST.COM for kudu/127.13.123.126@KRBTEST.COM
15:06:01.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.394286 13805 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:06:01.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:06:01.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/wal
15:06:01.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/data
15:06:01.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:01.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:01.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:01.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:01.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:01.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/logs
15:06:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/data/info.pb
15:06:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:06:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.123.124:43073
15:06:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.123.124
15:06:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.124.keytab
15:06:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.123.124
15:06:01.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:01.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:06:01.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:06:01.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.123.84:33031
15:06:01.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:01.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:01.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:01.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:01.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.123.126:46369,127.13.123.125:44729,127.13.123.124:43073
15:06:01.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:01.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42691/jwks.json
15:06:01.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:06:01.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.409485 13979 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } has no permanent_uuid. Determining permanent_uuid...
15:06:01.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:01 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521561, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.125@KRBTEST.COM for kudu/127.13.123.125@KRBTEST.COM
15:06:01.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.439644 13979 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } has no permanent_uuid. Determining permanent_uuid...
15:06:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:01.442520 13931 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.123.124:43073: connect: Connection refused (error 111)
15:06:01.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:01.446979 13979 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.123.124:43073: Network error: Client connection negotiation failed: client connection to 127.13.123.124:43073: connect: Connection refused (error 111)
15:06:01.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.496420 13979 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } attempt: 1
15:06:01.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:01.499949 13979 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.123.124:43073: Network error: Client connection negotiation failed: client connection to 127.13.123.124:43073: connect: Connection refused (error 111)
15:06:01.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.568331 13979 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } attempt: 2
15:06:01.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:01.572487 13979 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.123.124:43073: Network error: Client connection negotiation failed: client connection to 127.13.123.124:43073: connect: Connection refused (error 111)
15:06:01.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.663820 13979 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } attempt: 3
15:06:01.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:01.667343 13979 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.123.124:43073: Network error: Client connection negotiation failed: client connection to 127.13.123.124:43073: connect: Connection refused (error 111)
15:06:01.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.718545 13900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } attempt: 7
15:06:01.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:01 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521559, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.126@KRBTEST.COM for kudu/127.13.123.125@KRBTEST.COM
15:06:01.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.747289 13900 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } has no permanent_uuid. Determining permanent_uuid...
15:06:01.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:01.756850 13900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.123.124:43073: Network error: Client connection negotiation failed: client connection to 127.13.123.124:43073: connect: Connection refused (error 111)
15:06:01.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.810721 13979 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } attempt: 4
15:06:01.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.815546 13900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } attempt: 1
15:06:01.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:01.819145 13979 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.123.124:43073: Network error: Client connection negotiation failed: client connection to 127.13.123.124:43073: connect: Connection refused (error 111)
15:06:01.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:01.820897 13900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.123.124:43073: Network error: Client connection negotiation failed: client connection to 127.13.123.124:43073: connect: Connection refused (error 111)
15:06:01.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:01.842718 13985 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:01.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:01.843355 13985 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:01.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:01.844055 13985 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:01.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:01.874981 13985 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:06:01.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:01.875241 13985 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:06:01.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:01.875605 13985 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:01.875900 13985 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:06:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:01.876183 13985 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:06:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:01.876518 13985 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:06:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:01.876739 13985 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42691/jwks.json
15:06:01.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.902307 13900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } attempt: 2
15:06:01.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:01.909189 13900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.123.124:43073: Network error: Client connection negotiation failed: client connection to 127.13.123.124:43073: connect: Connection refused (error 111)
15:06:01.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.929888 13985 master_runner.cc:386] Master server non-default flags:
15:06:01.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:01.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.123.84:33031
15:06:01.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:01.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:01.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/data
15:06:01.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/wal
15:06:01.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:06:01.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:01.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.123.126:46369,127.13.123.125:44729,127.13.123.124:43073
15:06:01.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.124.keytab
15:06:01.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.123.124
15:06:01.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:01.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:01.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:01.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.123.124:43073
15:06:01.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:01.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:01.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:01.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42691/jwks.json
15:06:01.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:01.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:06:01.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:06:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/data/info.pb
15:06:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.123.124
15:06:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/logs
15:06:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:06:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:06:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:06:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 14:43:22 UTC on a633016ff5c9
15:06:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4112
15:06:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:01.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.931556 13985 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:01.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.932848 13985 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:01.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:01.952078 13998 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:01.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:01.994674 13900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } attempt: 3
15:06:02.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:02.001788 13900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.123.124:43073: Network error: Client connection negotiation failed: client connection to 127.13.123.124:43073: connect: Connection refused (error 111)
15:06:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:02.118597 13979 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } attempt: 5
15:06:02.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:02.125200 13979 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.123.124:43073: Network error: Client connection negotiation failed: client connection to 127.13.123.124:43073: connect: Connection refused (error 111)
15:06:02.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:02.142364 13900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } attempt: 4
15:06:02.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:02.148821 13900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.123.124:43073: Network error: Client connection negotiation failed: client connection to 127.13.123.124:43073: connect: Connection refused (error 111)
15:06:02.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:02.432358 13900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } attempt: 5
15:06:02.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:02.440454 13900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.123.124:43073: Network error: Client connection negotiation failed: client connection to 127.13.123.124:43073: connect: Connection refused (error 111)
15:06:02.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:02.672748 13979 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } attempt: 6
15:06:02.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:02.683382 13979 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.123.124:43073: Network error: Client connection negotiation failed: client connection to 127.13.123.124:43073: connect: Connection refused (error 111)
15:06:02.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:02.993077 13900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } attempt: 6
15:06:03.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:03.003207 13900 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.123.124:43073: Network error: Client connection negotiation failed: client connection to 127.13.123.124:43073: connect: Connection refused (error 111)
15:06:03.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:01.952913 13996 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:03.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:01.953126 13995 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:03.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:03.249557 13997 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1296 milliseconds
15:06:03.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.249670 13985 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:06:03.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.250864 13985 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:03.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.253281 13985 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:03.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.254599 13985 hybrid_clock.cc:648] HybridClock initialized: now 1736521563254574 us; error 42 us; skew 500 ppm
15:06:03.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.255375 13985 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:03.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:03 dist-test-slave-6prw krb5kdc[13817](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521563, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:03.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.261355 13985 init.cc:377] Logged in from keytab as kudu/127.13.123.124@KRBTEST.COM (short username kudu)
15:06:03.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.266793 13985 webserver.cc:458] Webserver started at http://127.13.123.124:36561/ using document root <none> and password file <none>
15:06:03.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.267638 13985 fs_manager.cc:362] Metadata directory not provided
15:06:03.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.267836 13985 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:03.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.268198 13985 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:06:03.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.272835 13985 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/data/instance:
15:06:03.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e698f9702ebe4cbdb156b67d1d5ad325"
15:06:03.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:06:03 on dist-test-slave-6prw"
15:06:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.273846 13985 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/wal/instance:
15:06:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e698f9702ebe4cbdb156b67d1d5ad325"
15:06:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:06:03 on dist-test-slave-6prw"
15:06:03.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.280987 13985 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
15:06:03.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.286028 14012 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:03.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.286998 13985 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.004s
15:06:03.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.287277 13985 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/data,/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/wal
15:06:03.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e698f9702ebe4cbdb156b67d1d5ad325"
15:06:03.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:06:03 on dist-test-slave-6prw"
15:06:03.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.287560 13985 fs_report.cc:389] FS layout report
15:06:03.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:03.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/wal
15:06:03.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/wal
15:06:03.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/data/data
15:06:03.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:03.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:03.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:03.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:03.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:03.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:03.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:03.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:03.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:03.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:03.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:03.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.345561 13985 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:03.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.350980 13985 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:03.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.351402 13985 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:03.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.428855 13985 rpc_server.cc:307] RPC server started. Bound to: 127.13.123.124:43073
15:06:03.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.428937 14063 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.123.124:43073 every 8 connection(s)
15:06:03.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.431474 13985 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/data/info.pb
15:06:03.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.437124 14064 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:06:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.440407 13805 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu as pid 13985
15:06:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.440730 13805 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/wal/instance
15:06:03.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.450829 14064 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } has no permanent_uuid. Determining permanent_uuid...
15:06:03.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:03 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521557, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.13.123.126@KRBTEST.COM
15:06:03.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:03 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521563, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.124@KRBTEST.COM for kudu/127.13.123.126@KRBTEST.COM
15:06:03.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.496547 14064 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } has no permanent_uuid. Determining permanent_uuid...
15:06:03.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:03 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521557, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.13.123.125@KRBTEST.COM
15:06:03.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:03 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521563, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.124@KRBTEST.COM for kudu/127.13.123.125@KRBTEST.COM
15:06:03.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:03 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521557, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.13.123.124@KRBTEST.COM
15:06:03.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.525425 14064 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } has no permanent_uuid. Determining permanent_uuid...
15:06:03.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:03 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521563, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.124@KRBTEST.COM for kudu/127.13.123.124@KRBTEST.COM
15:06:03.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.555987 14064 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325: Bootstrap starting.
15:06:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.562470 14064 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325: Neither blocks nor log segments found. Creating new log.
15:06:03.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.564105 14064 log.cc:826] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325: Log is configured to *not* fsync() on all Append() calls
15:06:03.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.569339 14064 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325: No bootstrap required, opened a new log
15:06:03.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.590341 14064 raft_consensus.cc:357] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } }
15:06:03.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.591100 14064 raft_consensus.cc:383] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:06:03.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.591382 14064 raft_consensus.cc:738] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e698f9702ebe4cbdb156b67d1d5ad325, State: Initialized, Role: FOLLOWER
15:06:03.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.592736 14064 consensus_queue.cc:260] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } }
15:06:03.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.595072 14076 sys_catalog.cc:455] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } } }
15:06:03.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.596159 14076 sys_catalog.cc:458] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [sys.catalog]: This master's current role is: FOLLOWER
15:06:03.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.596994 14064 sys_catalog.cc:564] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [sys.catalog]: configured and running, proceeding with master startup.
15:06:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:03.608225 14018 tablet.cc:2367] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:06:03.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:03.625965 14087 catalog_manager.cc:1559] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:06:03.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:03.626241 14087 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:06:03.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.629964 14076 raft_consensus.cc:491] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:06:03.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.630474 14076 raft_consensus.cc:513] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } }
15:06:03.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.633069 14076 leader_election.cc:290] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fbe595fcd2124f2b9c575098f0832994 (127.13.123.126:46369), 4c0b2ca5dec748b292bc8fc2a05e02e0 (127.13.123.125:44729)
15:06:03.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.622620 13985 master_runner.cc:186] Error getting master registration for 127.13.123.126:46369: OK, instance_id { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" instance_seqno: 1736521559329630 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:06:03.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.662995 13985 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:06:03.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.743919 13979 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } attempt: 7
15:06:03.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:03 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521561, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.125@KRBTEST.COM for kudu/127.13.123.124@KRBTEST.COM
15:06:03.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.782563 13979 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0: Bootstrap starting.
15:06:03.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.790462 13979 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0: Neither blocks nor log segments found. Creating new log.
15:06:03.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.792856 13979 log.cc:826] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0: Log is configured to *not* fsync() on all Append() calls
15:06:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.798019 13979 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0: No bootstrap required, opened a new log
15:06:03.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.821019 13979 raft_consensus.cc:357] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } }
15:06:03.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.821995 13979 raft_consensus.cc:383] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:06:03.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.822283 13979 raft_consensus.cc:738] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4c0b2ca5dec748b292bc8fc2a05e02e0, State: Initialized, Role: FOLLOWER
15:06:03.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.823163 13979 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } }
15:06:03.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.832278 14091 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } } }
15:06:03.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.834260 13979 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [sys.catalog]: configured and running, proceeding with master startup.
15:06:03.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.834398 13954 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" is_pre_election: true
15:06:03.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.835768 13954 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e698f9702ebe4cbdb156b67d1d5ad325 in term 0.
15:06:03.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.836297 14091 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [sys.catalog]: This master's current role is: FOLLOWER
15:06:03.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.837397 14015 leader_election.cc:304] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4c0b2ca5dec748b292bc8fc2a05e02e0, e698f9702ebe4cbdb156b67d1d5ad325; no voters:
15:06:03.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.838317 14076 raft_consensus.cc:2798] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:06:03.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.838635 14076 raft_consensus.cc:491] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:06:03.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.838943 14076 raft_consensus.cc:3054] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [term 0 FOLLOWER]: Advancing to term 1
15:06:03.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.846041 14076 raft_consensus.cc:513] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } }
15:06:03.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.850901 13954 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0"
15:06:03.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.852327 13954 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [term 0 FOLLOWER]: Advancing to term 1
15:06:03.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.860316 14076 leader_election.cc:290] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [CANDIDATE]: Term 1 election: Requested vote from peers fbe595fcd2124f2b9c575098f0832994 (127.13.123.126:46369), 4c0b2ca5dec748b292bc8fc2a05e02e0 (127.13.123.125:44729)
15:06:03.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.863139 13954 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e698f9702ebe4cbdb156b67d1d5ad325 in term 1.
15:06:03.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.864432 14015 leader_election.cc:304] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4c0b2ca5dec748b292bc8fc2a05e02e0, e698f9702ebe4cbdb156b67d1d5ad325; no voters:
15:06:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.865223 14076 raft_consensus.cc:2798] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [term 1 FOLLOWER]: Leader election won for term 1
15:06:03.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.874246 14076 raft_consensus.cc:695] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [term 1 LEADER]: Becoming Leader. State: Replica: e698f9702ebe4cbdb156b67d1d5ad325, State: Running, Role: LEADER
15:06:03.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.875639 14076 consensus_queue.cc:237] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } }
15:06:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.897505 14096 sys_catalog.cc:455] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e698f9702ebe4cbdb156b67d1d5ad325. Latest consensus state: current_term: 1 leader_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } } }
15:06:03.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.903522 14096 sys_catalog.cc:458] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [sys.catalog]: This master's current role is: LEADER
15:06:03.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.912539 14105 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
15:06:03.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.917124 14104 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:06:03.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.932588 14105 catalog_manager.cc:1485] Initializing Kudu cluster ID...
15:06:03.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.923202 13906 master_runner.cc:186] Error getting master registration for 127.13.123.126:46369: OK, instance_id { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" instance_seqno: 1736521559329630 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:06:03.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.958431 13906 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:06:03.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.966470 13954 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [term 1 FOLLOWER]: Refusing update from remote peer e698f9702ebe4cbdb156b67d1d5ad325: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:06:03.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:03.968904 14096 consensus_queue.cc:1035] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:06:03.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:03.988487 13933 tablet.cc:2367] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:06:04.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.004897 14091 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e698f9702ebe4cbdb156b67d1d5ad325. Latest consensus state: current_term: 1 leader_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } } }
15:06:04.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.007599 14091 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [sys.catalog]: This master's current role is: FOLLOWER
15:06:04.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.019896 14106 mvcc.cc:204] Tried to move back new op lower bound from 7112792325979082752 to 7112792325644111872. Current Snapshot: MvccSnapshot[applied={T|T < 7112792325979082752}]
15:06:04.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.020195 14091 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } } }
15:06:04.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.020924 14091 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [sys.catalog]: This master's current role is: FOLLOWER
15:06:04.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.029839 14105 catalog_manager.cc:1348] Generated new cluster ID: 471e1f0fb6504d9fafc5812ca92526b3
15:06:04.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.030583 14105 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
15:06:04.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.036790 13900 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } attempt: 7
15:06:04.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.041936 14096 sys_catalog.cc:455] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } } }
15:06:04.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.042613 14096 sys_catalog.cc:458] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [sys.catalog]: This master's current role is: LEADER
15:06:04.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:04 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521559, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.126@KRBTEST.COM for kudu/127.13.123.124@KRBTEST.COM
15:06:04.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.067641 14105 catalog_manager.cc:1371] Generated new certificate authority record
15:06:04.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.069595 14105 catalog_manager.cc:1505] Loading token signing keys...
15:06:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.078188 13900 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994: Bootstrap starting.
15:06:04.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.087357 13900 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994: Neither blocks nor log segments found. Creating new log.
15:06:04.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.089661 13900 log.cc:826] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994: Log is configured to *not* fsync() on all Append() calls
15:06:04.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.094928 13900 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994: No bootstrap required, opened a new log
15:06:04.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.098927 14105 catalog_manager.cc:5899] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325: Generated new TSK 0
15:06:04.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.100098 14105 catalog_manager.cc:1515] Initializing in-progress tserver states...
15:06:04.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.135038 13900 raft_consensus.cc:357] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } }
15:06:04.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.13.123.65@KRBTEST.COM; defaulting to no policy
15:06:04.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.137852 13900 raft_consensus.cc:383] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:06:04.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.138103 13900 raft_consensus.cc:738] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fbe595fcd2124f2b9c575098f0832994, State: Initialized, Role: FOLLOWER
15:06:04.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.139071 13900 consensus_queue.cc:260] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } }
15:06:04.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:04.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.13.123.65@KRBTEST.COM" created.
15:06:04.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.143263 13900 sys_catalog.cc:564] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [sys.catalog]: configured and running, proceeding with master startup.
15:06:04.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.143772 13873 raft_consensus.cc:3054] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [term 0 FOLLOWER]: Advancing to term 1
15:06:04.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.147720 13874 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fbe595fcd2124f2b9c575098f0832994"
15:06:04.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.152412 13875 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fbe595fcd2124f2b9c575098f0832994" is_pre_election: true
15:06:04.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.142292 14115 sys_catalog.cc:455] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } } }
15:06:04.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.154965 14115 sys_catalog.cc:458] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [sys.catalog]: This master's current role is: FOLLOWER
15:06:04.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.156466 13873 raft_consensus.cc:1270] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [term 1 FOLLOWER]: Refusing update from remote peer e698f9702ebe4cbdb156b67d1d5ad325: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:06:04.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.158111 14096 consensus_queue.cc:1035] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:06:04.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:04.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.123.65 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.65.keytab.
15:06:04.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.123.65 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.65.keytab.
15:06:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.174916 13805 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:06:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:06:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-0/wal
15:06:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-0/data
15:06:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-0/logs
15:06:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-0/data/info.pb
15:06:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:04.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:04.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:06:04.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:04.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.123.65:0
15:06:04.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.123.65
15:06:04.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.123.65
15:06:04.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:04.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.123.126:46369,127.13.123.125:44729,127.13.123.124:43073
15:06:04.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.65.keytab
15:06:04.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.123.65
15:06:04.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:04.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:06:04.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:06:04.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.123.84:33031
15:06:04.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:04.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:04.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {KRB5CCNAME=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:06:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.290743 14115 sys_catalog.cc:455] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e698f9702ebe4cbdb156b67d1d5ad325. Latest consensus state: current_term: 1 leader_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } } }
15:06:04.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.291471 14115 sys_catalog.cc:458] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [sys.catalog]: This master's current role is: FOLLOWER
15:06:04.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.292342 14130 sys_catalog.cc:455] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } } }
15:06:04.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.293309 14130 sys_catalog.cc:458] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [sys.catalog]: This master's current role is: FOLLOWER
15:06:04.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.305635 14133 catalog_manager.cc:1260] Loaded cluster ID: 471e1f0fb6504d9fafc5812ca92526b3
15:06:04.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.306157 14133 catalog_manager.cc:1553] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994: loading cluster ID for follower catalog manager: success
15:06:04.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.318713 14133 catalog_manager.cc:1575] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994: acquiring CA information for follower catalog manager: success
15:06:04.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.325101 14133 catalog_manager.cc:1603] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:06:04.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.725039 14096 sys_catalog.cc:455] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } } }
15:06:04.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.727263 14096 sys_catalog.cc:458] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [sys.catalog]: This master's current role is: LEADER
15:06:04.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:04.749500 14123 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:04.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:04.750008 14123 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:04.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:04.750691 14123 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:04.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:04.793696 14123 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:04.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:04.795650 14123 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.123.65
15:06:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.835795 14123 tablet_server_runner.cc:78] Tablet server non-default flags:
15:06:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.123.84:33031
15:06:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-0/data
15:06:04.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-0/wal
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.65.keytab
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.123.65
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.123.65:0
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-0/data/info.pb
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.123.65
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.123.126:46369,127.13.123.125:44729,127.13.123.124:43073
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.123.65
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-0/logs
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:04.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:06:04.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:06:04.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:06:04.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:04.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 14:43:22 UTC on a633016ff5c9
15:06:04.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4112
15:06:04.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:04.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.837388 14123 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:04.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.838722 14123 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:04.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:04.858286 14143 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:04.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.938216 14104 catalog_manager.cc:1260] Loaded cluster ID: 471e1f0fb6504d9fafc5812ca92526b3
15:06:04.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.938819 14104 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0: loading cluster ID for follower catalog manager: success
15:06:04.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.957710 14104 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0: acquiring CA information for follower catalog manager: success
15:06:04.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:04.970523 14104 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:06:05.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:05.245571 13975 debug-util.cc:398] Leaking SignalData structure 0x7b0800084260 after lost signal to thread 13909
15:06:05.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:05.246266 13975 debug-util.cc:398] Leaking SignalData structure 0x7b08000850a0 after lost signal to thread 13978
15:06:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:06.254853 14142 debug-util.cc:398] Leaking SignalData structure 0x7b080001ca40 after lost signal to thread 14123
15:06:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:08.727273 14123 thread.cc:635] OpenStack (cloud detector) Time spent creating pthread: real 3.867s user 0.709s sys 1.314s
15:06:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:08.727658 14123 thread.cc:601] OpenStack (cloud detector) Time spent starting thread: real 3.867s user 0.709s sys 1.314s
15:06:08.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:04.860378 14144 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:08.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:08.730084 14145 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3869 milliseconds
15:06:08.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:08.731244 14146 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:08.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.731175 14123 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:06:08.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.735925 14123 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:08.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.752502 14123 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:08.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.768337 14123 hybrid_clock.cc:648] HybridClock initialized: now 1736521568768270 us; error 65 us; skew 500 ppm
15:06:08.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.769490 14123 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:08.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:08 dist-test-slave-6prw krb5kdc[13817](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521568, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:08.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.777626 14123 init.cc:377] Logged in from keytab as kudu/127.13.123.65@KRBTEST.COM (short username kudu)
15:06:08.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.810583 14123 webserver.cc:458] Webserver started at http://127.13.123.65:37343/ using document root <none> and password file <none>
15:06:08.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.812111 14123 fs_manager.cc:362] Metadata directory not provided
15:06:08.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.812611 14123 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:08.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.813264 14123 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:06:08.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.820592 14123 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-0/data/instance:
15:06:08.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "342c965824f04206b4a5938b066b652f"
15:06:08.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:06:08 on dist-test-slave-6prw"
15:06:08.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.822263 14123 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-0/wal/instance:
15:06:08.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "342c965824f04206b4a5938b066b652f"
15:06:08.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:06:08 on dist-test-slave-6prw"
15:06:08.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.838084 14123 fs_manager.cc:696] Time spent creating directory manager: real 0.015s user 0.009s sys 0.005s
15:06:08.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.851419 14164 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:08.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.853077 14123 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.004s sys 0.004s
15:06:08.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.853636 14123 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-0/data,/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-0/wal
15:06:08.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "342c965824f04206b4a5938b066b652f"
15:06:08.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:06:08 on dist-test-slave-6prw"
15:06:08.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.854281 14123 fs_report.cc:389] FS layout report
15:06:08.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-0/wal
15:06:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-0/wal
15:06:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-0/data/data
15:06:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:08.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.978875 14123 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:08.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.986192 14123 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:08.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.986766 14123 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:08.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.989840 14123 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:06:08.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.996848 14123 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:06:08.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.997155 14123 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:06:08.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.997466 14123 ts_tablet_manager.cc:610] Registered 0 tablets
15:06:08.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:08.997695 14123 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:06:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:09 dist-test-slave-6prw krb5kdc[13817](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521569, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:09.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:09.199744 13844 init.cc:295] Successfully reacquired a new kerberos TGT
15:06:09.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:09.228965 14123 rpc_server.cc:307] RPC server started. Bound to: 127.13.123.65:42801
15:06:09.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:09.229569 14276 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.123.65:42801 every 8 connection(s)
15:06:09.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:09.232445 14123 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-0/data/info.pb
15:06:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:09.233219 13805 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu as pid 14123
15:06:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:09.233737 13805 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-0/wal/instance
15:06:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.13.123.66@KRBTEST.COM; defaulting to no policy
15:06:09.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:09.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.13.123.66@KRBTEST.COM" created.
15:06:09.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:09 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521568, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.65@KRBTEST.COM for kudu/127.13.123.124@KRBTEST.COM
15:06:09.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:09 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521568, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.65@KRBTEST.COM for kudu/127.13.123.126@KRBTEST.COM
15:06:09.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:09 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521568, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.65@KRBTEST.COM for kudu/127.13.123.125@KRBTEST.COM
15:06:09.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:09.333117 14277 heartbeater.cc:346] Connected to a master server at 127.13.123.125:44729
15:06:09.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:09.333626 14277 heartbeater.cc:463] Registering TS with master...
15:06:09.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:09.335103 14277 heartbeater.cc:510] Master 127.13.123.125:44729 requested a full tablet report, sending...
15:06:09.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:09.336423 14279 heartbeater.cc:346] Connected to a master server at 127.13.123.126:46369
15:06:09.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:09.336797 14279 heartbeater.cc:463] Registering TS with master...
15:06:09.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:09.337793 14279 heartbeater.cc:510] Master 127.13.123.126:46369 requested a full tablet report, sending...
15:06:09.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:09.339262 14278 heartbeater.cc:346] Connected to a master server at 127.13.123.124:43073
15:06:09.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:09.339601 14278 heartbeater.cc:463] Registering TS with master...
15:06:09.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:09.340512 14278 heartbeater.cc:510] Master 127.13.123.124:43073 requested a full tablet report, sending...
15:06:09.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:09.342008 13865 ts_manager.cc:194] Registered new tserver with Master: 342c965824f04206b4a5938b066b652f (127.13.123.65:42801)
15:06:09.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:09.342120 13944 ts_manager.cc:194] Registered new tserver with Master: 342c965824f04206b4a5938b066b652f (127.13.123.65:42801)
15:06:09.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:09.344841 14029 ts_manager.cc:194] Registered new tserver with Master: 342c965824f04206b4a5938b066b652f (127.13.123.65:42801)
15:06:09.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:09.346843 14029 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.13.123.65@KRBTEST.COM'} at 127.13.123.65:53799
15:06:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.123.66 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.66.keytab.
15:06:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.123.66 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.66.keytab.
15:06:09.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:09.353026 13805 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:06:09.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:06:09.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-1/wal
15:06:09.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-1/data
15:06:09.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:09.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:09.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:09.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:09.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:09.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:09.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-1/logs
15:06:09.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-1/data/info.pb
15:06:09.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:09.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:09.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:09.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:09.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:09.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:09.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:09.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:09.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:06:09.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:09.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.123.66:0
15:06:09.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.123.66
15:06:09.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.123.66
15:06:09.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:09.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.123.126:46369,127.13.123.125:44729,127.13.123.124:43073
15:06:09.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.66.keytab
15:06:09.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.123.66
15:06:09.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:09.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:06:09.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:06:09.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.123.84:33031
15:06:09.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:09.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:09.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {KRB5CCNAME=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:06:09.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:09.771605 14289 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:09.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:09.772264 14289 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:09.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:09.773200 14289 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:09.808521 14289 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:09.809441 14289 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.123.66
15:06:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:09.848012 14289 tablet_server_runner.cc:78] Tablet server non-default flags:
15:06:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.123.84:33031
15:06:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-1/data
15:06:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-1/wal
15:06:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.66.keytab
15:06:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.123.66
15:06:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.123.66:0
15:06:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-1/data/info.pb
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.123.66
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.123.126:46369,127.13.123.125:44729,127.13.123.124:43073
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.123.66
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-1/logs
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 14:43:22 UTC on a633016ff5c9
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4112
15:06:09.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:09.849699 14289 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:09.851078 14289 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:09.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:09.871428 14298 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:10.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:10.350224 14278 heartbeater.cc:502] Master 127.13.123.124:43073 was elected leader, sending a full tablet report...
15:06:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:09.873366 14300 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:11.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:09.872579 14297 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:11.085209 14299 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
15:06:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.085275 14289 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:06:11.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.089666 14289 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:11.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.091786 14289 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:11.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.093130 14289 hybrid_clock.cc:648] HybridClock initialized: now 1736521571093103 us; error 55 us; skew 500 ppm
15:06:11.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.093901 14289 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:11.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:11 dist-test-slave-6prw krb5kdc[13817](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521571, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:11.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.100296 14289 init.cc:377] Logged in from keytab as kudu/127.13.123.66@KRBTEST.COM (short username kudu)
15:06:11.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.105297 14289 webserver.cc:458] Webserver started at http://127.13.123.66:44253/ using document root <none> and password file <none>
15:06:11.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.106251 14289 fs_manager.cc:362] Metadata directory not provided
15:06:11.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.106469 14289 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:11.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.107081 14289 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:06:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:11 dist-test-slave-6prw krb5kdc[13817](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521571, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.111914 13923 init.cc:295] Successfully reacquired a new kerberos TGT
15:06:11.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.112920 14289 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-1/data/instance:
15:06:11.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82402bf20ce84da08e47cd79295ab879"
15:06:11.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:06:11 on dist-test-slave-6prw"
15:06:11.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.114343 14289 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-1/wal/instance:
15:06:11.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82402bf20ce84da08e47cd79295ab879"
15:06:11.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:06:11 on dist-test-slave-6prw"
15:06:11.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.123579 14289 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.006s sys 0.004s
15:06:11.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.129483 14308 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:11.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.130558 14289 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
15:06:11.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.130924 14289 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-1/data,/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-1/wal
15:06:11.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82402bf20ce84da08e47cd79295ab879"
15:06:11.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:06:11 on dist-test-slave-6prw"
15:06:11.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.131292 14289 fs_report.cc:389] FS layout report
15:06:11.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:11.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-1/wal
15:06:11.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-1/wal
15:06:11.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-1/data/data
15:06:11.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:11.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:11.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:11.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:11.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:11.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:11.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:11.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:11.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:11.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:11.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:11.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.178942 14289 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:11.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.184911 14289 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:11.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.185389 14289 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:11.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.188917 14289 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:06:11.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.193483 14289 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:06:11.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.193776 14289 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:06:11.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.193980 14289 ts_tablet_manager.cc:610] Registered 0 tablets
15:06:11.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.194116 14289 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:06:11.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.347429 14289 rpc_server.cc:307] RPC server started. Bound to: 127.13.123.66:40567
15:06:11.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.347522 14420 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.123.66:40567 every 8 connection(s)
15:06:11.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.350593 14289 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-1/data/info.pb
15:06:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.352490 13805 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu as pid 14289
15:06:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.352946 13805 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-1/wal/instance
15:06:11.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.13.123.67@KRBTEST.COM; defaulting to no policy
15:06:11.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:11 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521571, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.66@KRBTEST.COM for kudu/127.13.123.125@KRBTEST.COM
15:06:11.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:11 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521571, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.66@KRBTEST.COM for kudu/127.13.123.124@KRBTEST.COM
15:06:11.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:11.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.13.123.67@KRBTEST.COM" created.
15:06:11.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:11 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521571, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.66@KRBTEST.COM for kudu/127.13.123.126@KRBTEST.COM
15:06:11.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.443346 14421 heartbeater.cc:346] Connected to a master server at 127.13.123.125:44729
15:06:11.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.443825 14421 heartbeater.cc:463] Registering TS with master...
15:06:11.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.445142 14421 heartbeater.cc:510] Master 127.13.123.125:44729 requested a full tablet report, sending...
15:06:11.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.448289 13944 ts_manager.cc:194] Registered new tserver with Master: 82402bf20ce84da08e47cd79295ab879 (127.13.123.66:40567)
15:06:11.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.453508 14423 heartbeater.cc:346] Connected to a master server at 127.13.123.126:46369
15:06:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.453886 14423 heartbeater.cc:463] Registering TS with master...
15:06:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.454669 14423 heartbeater.cc:510] Master 127.13.123.126:46369 requested a full tablet report, sending...
15:06:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.457267 13865 ts_manager.cc:194] Registered new tserver with Master: 82402bf20ce84da08e47cd79295ab879 (127.13.123.66:40567)
15:06:11.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.458371 14422 heartbeater.cc:346] Connected to a master server at 127.13.123.124:43073
15:06:11.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.458701 14422 heartbeater.cc:463] Registering TS with master...
15:06:11.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.459558 14422 heartbeater.cc:510] Master 127.13.123.124:43073 requested a full tablet report, sending...
15:06:11.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.461611 14028 ts_manager.cc:194] Registered new tserver with Master: 82402bf20ce84da08e47cd79295ab879 (127.13.123.66:40567)
15:06:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.463001 14028 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.13.123.66@KRBTEST.COM'} at 127.13.123.66:50655
15:06:11.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.123.67 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.67.keytab.
15:06:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.123.67 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.67.keytab.
15:06:11.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:11.474548 13805 mini_kdc.cc:262] Time spent creating service keytab for kudu/127.13.123.67: real 0.107s user 0.004s sys 0.010s
15:06:11.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.476608 13805 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-2/wal
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-2/data
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-2/logs
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-2/data/info.pb
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.123.67:0
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.123.67
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.123.67
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.123.126:46369,127.13.123.125:44729,127.13.123.124:43073
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.67.keytab
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.123.67
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:06:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.123.84:33031
15:06:11.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:11.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:11.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {KRB5CCNAME=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:06:11.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:11.798854 14433 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:11.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:11.799500 14433 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:11.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:11.800375 14433 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:11.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:11.836082 14433 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:11.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:11.837513 14433 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.123.67
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.875054 14433 tablet_server_runner.cc:78] Tablet server non-default flags:
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.123.84:33031
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-2/data
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-2/wal
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.67.keytab
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.123.67
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.123.67:0
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-2/data/info.pb
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.123.67
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.123.126:46369,127.13.123.125:44729,127.13.123.124:43073
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.123.67
15:06:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-2/logs
15:06:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:06:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:06:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:06:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 14:43:22 UTC on a633016ff5c9
15:06:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4112
15:06:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.876343 14433 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.877285 14433 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:11.894815 14440 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:12.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:12.466300 14422 heartbeater.cc:502] Master 127.13.123.124:43073 was elected leader, sending a full tablet report...
15:06:13.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:11.895489 14441 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:13.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:11.896384 14443 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:13.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:11.897150 14433 server_base.cc:1034] running on GCE node
15:06:13.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.129796 14433 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:13.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.132467 14433 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:13.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.133913 14433 hybrid_clock.cc:648] HybridClock initialized: now 1736521573133886 us; error 39 us; skew 500 ppm
15:06:13.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.134672 14433 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:13.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:13 dist-test-slave-6prw krb5kdc[13817](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521573, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:13.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.170748 14433 init.cc:377] Logged in from keytab as kudu/127.13.123.67@KRBTEST.COM (short username kudu)
15:06:13.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.176507 14433 webserver.cc:458] Webserver started at http://127.13.123.67:40091/ using document root <none> and password file <none>
15:06:13.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.177415 14433 fs_manager.cc:362] Metadata directory not provided
15:06:13.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.177636 14433 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:13.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.178129 14433 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:06:13.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.183094 14433 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-2/data/instance:
15:06:13.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5c08693f59ea487391477364ecf7f0f5"
15:06:13.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:06:13 on dist-test-slave-6prw"
15:06:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.184209 14433 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-2/wal/instance:
15:06:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5c08693f59ea487391477364ecf7f0f5"
15:06:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:06:13 on dist-test-slave-6prw"
15:06:13.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.191709 14433 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
15:06:13.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.197371 14453 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.198410 14433 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
15:06:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.198794 14433 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-2/data,/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-2/wal
15:06:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5c08693f59ea487391477364ecf7f0f5"
15:06:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:06:13 on dist-test-slave-6prw"
15:06:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.199102 14433 fs_report.cc:389] FS layout report
15:06:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-2/wal
15:06:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-2/wal
15:06:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-2/data/data
15:06:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:13.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:13.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.252487 14433 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:13.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.257983 14433 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:13.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.258396 14433 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:13.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.261240 14433 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:06:13.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:13 dist-test-slave-6prw krb5kdc[13817](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521573, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:13.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.265255 14008 init.cc:295] Successfully reacquired a new kerberos TGT
15:06:13.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.266580 14433 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:06:13.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.266777 14433 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:06:13.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.266995 14433 ts_tablet_manager.cc:610] Registered 0 tablets
15:06:13.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.267115 14433 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:06:13.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.435467 14433 rpc_server.cc:307] RPC server started. Bound to: 127.13.123.67:40771
15:06:13.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.435603 14565 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.123.67:40771 every 8 connection(s)
15:06:13.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.438423 14433 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-2/data/info.pb
15:06:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.448033 13805 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu as pid 14433
15:06:13.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.448658 13805 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/ts-2/wal/instance
15:06:13.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:13 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521573, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.67@KRBTEST.COM for kudu/127.13.123.125@KRBTEST.COM
15:06:13.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:13 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521573, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.67@KRBTEST.COM for kudu/127.13.123.124@KRBTEST.COM
15:06:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.532563 14566 heartbeater.cc:346] Connected to a master server at 127.13.123.125:44729
15:06:13.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.533046 14566 heartbeater.cc:463] Registering TS with master...
15:06:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.534723 14566 heartbeater.cc:510] Master 127.13.123.125:44729 requested a full tablet report, sending...
15:06:13.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.539574 13944 ts_manager.cc:194] Registered new tserver with Master: 5c08693f59ea487391477364ecf7f0f5 (127.13.123.67:40771)
15:06:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:13 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521573, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.67@KRBTEST.COM for kudu/127.13.123.126@KRBTEST.COM
15:06:13.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.545993 14567 heartbeater.cc:346] Connected to a master server at 127.13.123.124:43073
15:06:13.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.546347 14567 heartbeater.cc:463] Registering TS with master...
15:06:13.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.547294 14567 heartbeater.cc:510] Master 127.13.123.124:43073 requested a full tablet report, sending...
15:06:13.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.549229 14028 ts_manager.cc:194] Registered new tserver with Master: 5c08693f59ea487391477364ecf7f0f5 (127.13.123.67:40771)
15:06:13.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.550645 14028 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.13.123.67@KRBTEST.COM'} at 127.13.123.67:33001
15:06:13.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.556543 14570 heartbeater.cc:346] Connected to a master server at 127.13.123.126:46369
15:06:13.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.557004 14570 heartbeater.cc:463] Registering TS with master...
15:06:13.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.557585 14570 heartbeater.cc:510] Master 127.13.123.126:46369 requested a full tablet report, sending...
15:06:13.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.560189 13864 ts_manager.cc:194] Registered new tserver with Master: 5c08693f59ea487391477364ecf7f0f5 (127.13.123.67:40771)
15:06:13.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.564603 13805 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
15:06:13.578 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:13.580 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
15:06:13.592 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
15:06:13.593 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
15:06:13.596 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.13.123.126"
port: 46369
}
bound_http_address {
host: "127.13.123.126"
port: 38515
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.13.123.125"
port: 44729
}
bound_http_address {
host: "127.13.123.125"
port: 35827
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.13.123.124"
port: 43073
}
bound_http_address {
host: "127.13.123.124"
port: 36561
}
}
}
15:06:13.597 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:06:13.598 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.13.123.65"
port: 42801
}
bound_http_address {
host: "127.13.123.65"
port: 37343
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.13.123.66"
port: 40567
}
bound_http_address {
host: "127.13.123.66"
port: 44253
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.13.123.67"
port: 40771
}
bound_http_address {
host: "127.13.123.67"
port: 40091
}
}
}
15:06:13.598 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:06:13.600 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:06:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:13 dist-test-slave-6prw krb5kdc[13817](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521573, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:13.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:06:13.639 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:13.641 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/krb5cc
15:06:13.648 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:06:13.648 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/krb5cc
15:06:13.649 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:06:13.653 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
15:06:13.654 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:06:13.654 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:06:13.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:13 dist-test-slave-6prw krb5kdc[13817](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521573, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:13.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:06:13.693 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:13.693 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/krb5cc
15:06:13.695 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:06:13.695 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.13.123.125:44729
15:06:13.697 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
15:06:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.698189 13805 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu with pid 13906
15:06:13.733 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:13.733 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.13.123.126:46369
15:06:13.733 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
15:06:13.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.734076 13805 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu with pid 13831
15:06:13.764 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:13.764 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.13.123.124:43073
15:06:13.764 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
15:06:13.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.765002 13805 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu with pid 13985
15:06:13.792 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:13.793 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.13.123.125:44729
15:06:13.797 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
15:06:13.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:13.798585 13805 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/wal
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/data
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/logs
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/data/info.pb
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.123.125:44729
15:06:13.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.123.125
15:06:13.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35827
15:06:13.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.125.keytab
15:06:13.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.123.125
15:06:13.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:13.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:06:13.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:06:13.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.123.84:33031
15:06:13.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:13.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:13.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:13.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:13.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.123.126:46369,127.13.123.125:44729,127.13.123.124:43073
15:06:13.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:13.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42691/jwks.json
15:06:13.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:06:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:14.107744 14581 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:14.108352 14581 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:14.109130 14581 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:14.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:14.141518 14581 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:06:14.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:14.141762 14581 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:06:14.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:14.142140 14581 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:14.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:14.142376 14581 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:06:14.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:14.142572 14581 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:06:14.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:14.142764 14581 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:06:14.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:14.142906 14581 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42691/jwks.json
15:06:14.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:14.179246 14581 master_runner.cc:386] Master server non-default flags:
15:06:14.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:14.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.123.84:33031
15:06:14.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:14.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:14.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/data
15:06:14.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/wal
15:06:14.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.123.126:46369,127.13.123.125:44729,127.13.123.124:43073
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.125.keytab
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.123.125
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.123.125:44729
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42691/jwks.json
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/data/info.pb
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.123.125
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35827
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/logs
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 14:43:22 UTC on a633016ff5c9
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4112
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:14.180570 14581 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:14.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:14.181900 14581 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:14.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:14.198477 14588 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:14.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:14.369874 14277 heartbeater.cc:643] Failed to heartbeat to 127.13.123.125:44729 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.13.123.125:44729: connect: Connection refused (error 111)
15:06:14.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:14.461422 14421 heartbeater.cc:643] Failed to heartbeat to 127.13.123.125:44729 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.13.123.125:44729: connect: Connection refused (error 111)
15:06:14.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:14.545917 14566 heartbeater.cc:643] Failed to heartbeat to 127.13.123.125:44729 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.13.123.125:44729: connect: Connection refused (error 111)
15:06:14.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:14.554517 14567 heartbeater.cc:502] Master 127.13.123.124:43073 was elected leader, sending a full tablet report...
15:06:15.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:14.198843 14589 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:15.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:14.201047 14581 server_base.cc:1034] running on GCE node
15:06:15.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:14.200353 14591 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:15.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.442373 14581 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:15.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.445405 14581 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:15.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.446828 14581 hybrid_clock.cc:648] HybridClock initialized: now 1736521575446788 us; error 56 us; skew 500 ppm
15:06:15.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.447633 14581 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:15.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:15 dist-test-slave-6prw krb5kdc[13817](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521575, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:15.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.453501 14581 init.cc:377] Logged in from keytab as kudu/127.13.123.125@KRBTEST.COM (short username kudu)
15:06:15.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.458599 14581 webserver.cc:458] Webserver started at http://127.13.123.125:35827/ using document root <none> and password file <none>
15:06:15.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.459506 14581 fs_manager.cc:362] Metadata directory not provided
15:06:15.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.459719 14581 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:15.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.468425 14581 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.003s sys 0.005s
15:06:15.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.473920 14607 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:15.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.475425 14581 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
15:06:15.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.475826 14581 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/data,/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/wal
15:06:15.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0"
15:06:15.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:06:01 on dist-test-slave-6prw"
15:06:15.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.479176 14581 fs_report.cc:389] FS layout report
15:06:15.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:15.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/wal
15:06:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/wal
15:06:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/data/data
15:06:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:15.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.554242 14581 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:15.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.560063 14581 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:15.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.560580 14581 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:15.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.638609 14581 rpc_server.cc:307] RPC server started. Bound to: 127.13.123.125:44729
15:06:15.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.638720 14659 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.123.125:44729 every 8 connection(s)
15:06:15.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.642706 14581 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-1/data/info.pb
15:06:15.652 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:15.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.652086 13805 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu as pid 14581
15:06:15.653 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.13.123.126:46369
15:06:15.654 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
15:06:15.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.655851 13805 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:06:15.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:06:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/wal
15:06:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/data
15:06:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/logs
15:06:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/data/info.pb
15:06:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:06:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:15.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:15.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:15.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.123.126:46369
15:06:15.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.123.126
15:06:15.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38515
15:06:15.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.126.keytab
15:06:15.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.123.126
15:06:15.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:15.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:06:15.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:06:15.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.123.84:33031
15:06:15.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:15.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:15.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:15.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:15.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.123.126:46369,127.13.123.125:44729,127.13.123.124:43073
15:06:15.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:15.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42691/jwks.json
15:06:15.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:06:15.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.662890 14660 sys_catalog.cc:263] Verifying existing consensus state
15:06:15.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.673565 14660 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0: Bootstrap starting.
15:06:15.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.717984 14660 log.cc:826] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0: Log is configured to *not* fsync() on all Append() calls
15:06:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.732388 14660 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:06:15.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.733107 14660 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0: Bootstrap complete.
15:06:15.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.756561 14660 raft_consensus.cc:357] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } }
15:06:15.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.758808 14660 raft_consensus.cc:738] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4c0b2ca5dec748b292bc8fc2a05e02e0, State: Initialized, Role: FOLLOWER
15:06:15.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.759548 14660 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } }
15:06:15.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.762315 14664 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } } }
15:06:15.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.763159 14664 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [sys.catalog]: This master's current role is: FOLLOWER
15:06:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.764135 14660 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [sys.catalog]: configured and running, proceeding with master startup.
15:06:15.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.793093 14675 catalog_manager.cc:1260] Loaded cluster ID: 471e1f0fb6504d9fafc5812ca92526b3
15:06:15.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.793352 14675 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0: loading cluster ID for follower catalog manager: success
15:06:15.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.801527 14675 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0: acquiring CA information for follower catalog manager: success
15:06:15.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.807511 14675 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:06:15.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.807017 14581 master_runner.cc:186] Error getting master registration for 127.13.123.126:46369: Network error: Client connection negotiation failed: client connection to 127.13.123.126:46369: connect: Connection refused (error 111),
15:06:15.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:15.841076 14581 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:06:16.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:16.002727 14662 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:16.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:16.003484 14662 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:16.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:16.004570 14662 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:16.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:16.063362 14662 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:06:16.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:16.063643 14662 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:06:16.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:16.064110 14662 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:16.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:16.064455 14662 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:06:16.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:16.064754 14662 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:06:16.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:16.065052 14662 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:06:16.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:16.065294 14662 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42691/jwks.json
15:06:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:16.106118 14662 master_runner.cc:386] Master server non-default flags:
15:06:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.123.84:33031
15:06:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/data
15:06:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/wal
15:06:16.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:06:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.123.126:46369,127.13.123.125:44729,127.13.123.124:43073
15:06:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.126.keytab
15:06:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.123.126
15:06:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.123.126:46369
15:06:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42691/jwks.json
15:06:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:06:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:06:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/data/info.pb
15:06:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.123.126
15:06:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38515
15:06:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:16.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:16.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:16.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:16.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:16.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/logs
15:06:16.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:16.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:16.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:16.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:06:16.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:06:16.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:06:16.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:16.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 14:43:22 UTC on a633016ff5c9
15:06:16.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4112
15:06:16.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:16.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:16.107800 14662 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:16.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:16.109150 14662 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:16.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:16.125310 14684 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:16.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:16.412094 14277 heartbeater.cc:346] Connected to a master server at 127.13.123.125:44729
15:06:16.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:16.417230 14625 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "342c965824f04206b4a5938b066b652f" instance_seqno: 1736521569176882) as {username='kudu', principal='kudu/127.13.123.65@KRBTEST.COM'} at 127.13.123.65:51321; Asking this server to re-register.
15:06:16.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:16.418499 14277 heartbeater.cc:463] Registering TS with master...
15:06:16.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:16.421074 14625 ts_manager.cc:194] Registered new tserver with Master: 342c965824f04206b4a5938b066b652f (127.13.123.65:42801)
15:06:16.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:16.499344 14421 heartbeater.cc:346] Connected to a master server at 127.13.123.125:44729
15:06:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:16.501729 14625 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "82402bf20ce84da08e47cd79295ab879" instance_seqno: 1736521571314526) as {username='kudu', principal='kudu/127.13.123.66@KRBTEST.COM'} at 127.13.123.66:35649; Asking this server to re-register.
15:06:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:16.503012 14421 heartbeater.cc:463] Registering TS with master...
15:06:16.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:16.505853 14625 ts_manager.cc:194] Registered new tserver with Master: 82402bf20ce84da08e47cd79295ab879 (127.13.123.66:40567)
15:06:16.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:16.565797 14566 heartbeater.cc:346] Connected to a master server at 127.13.123.125:44729
15:06:16.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:16.569517 14625 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "5c08693f59ea487391477364ecf7f0f5" instance_seqno: 1736521573399616) as {username='kudu', principal='kudu/127.13.123.67@KRBTEST.COM'} at 127.13.123.67:44471; Asking this server to re-register.
15:06:16.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:16.572047 14566 heartbeater.cc:463] Registering TS with master...
15:06:16.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:16.576682 14625 ts_manager.cc:194] Registered new tserver with Master: 5c08693f59ea487391477364ecf7f0f5 (127.13.123.67:40771)
15:06:16.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:16.649192 13232 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac300 after lost signal to thread 13167
15:06:16.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:16.650153 13232 debug-util.cc:398] Leaking SignalData structure 0x7b0800090d40 after lost signal to thread 13235
15:06:17.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:17.070065 13790 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8960 after lost signal to thread 13665
15:06:17.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:17.071573 13790 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8e60 after lost signal to thread 13793
15:06:17.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:16.129470 14662 server_base.cc:1034] running on GCE node
15:06:17.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:16.130234 14686 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:17.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:16.128294 14683 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:17.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:17.623884 14662 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:17.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:17.633045 14662 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:17.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:17.634464 14706 raft_consensus.cc:491] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:06:17.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:17.635301 14706 raft_consensus.cc:513] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } }
15:06:17.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:17.640853 14662 hybrid_clock.cc:648] HybridClock initialized: now 1736521577640772 us; error 83 us; skew 500 ppm
15:06:17.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:17.642004 14662 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:17.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:17 dist-test-slave-6prw krb5kdc[13817](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521577, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:17.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:17.654567 14662 init.cc:377] Logged in from keytab as kudu/127.13.123.126@KRBTEST.COM (short username kudu)
15:06:17.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:17.659880 14706 leader_election.cc:290] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers fbe595fcd2124f2b9c575098f0832994 (127.13.123.126:46369), e698f9702ebe4cbdb156b67d1d5ad325 (127.13.123.124:43073)
15:06:17.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:17.666694 14611 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.123.124:43073: connect: Connection refused (error 111)
15:06:17.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:17.675366 14611 leader_election.cc:336] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer e698f9702ebe4cbdb156b67d1d5ad325 (127.13.123.124:43073): Network error: Client connection negotiation failed: client connection to 127.13.123.124:43073: connect: Connection refused (error 111)
15:06:17.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:17.688846 14610 leader_election.cc:336] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer fbe595fcd2124f2b9c575098f0832994 (127.13.123.126:46369): Network error: Client connection negotiation failed: client connection to 127.13.123.126:46369: connect: Connection refused (error 111)
15:06:17.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:17.689561 14610 leader_election.cc:304] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4c0b2ca5dec748b292bc8fc2a05e02e0; no voters: e698f9702ebe4cbdb156b67d1d5ad325, fbe595fcd2124f2b9c575098f0832994
15:06:17.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:17.690624 14706 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
15:06:17.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:17.692574 14662 webserver.cc:458] Webserver started at http://127.13.123.126:38515/ using document root <none> and password file <none>
15:06:17.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:17.694067 14662 fs_manager.cc:362] Metadata directory not provided
15:06:17.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:17.694512 14662 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:17.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:17.712275 14662 fs_manager.cc:714] Time spent opening directory manager: real 0.013s user 0.005s sys 0.008s
15:06:17.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:17.724349 14715 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:17.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:17.726106 14662 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.008s sys 0.000s
15:06:17.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:17.726725 14662 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/data,/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/wal
15:06:17.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fbe595fcd2124f2b9c575098f0832994"
15:06:17.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:05:59 on dist-test-slave-6prw"
15:06:17.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:17.730985 14662 fs_report.cc:389] FS layout report
15:06:17.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:17.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/wal
15:06:17.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/wal
15:06:17.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/data/data
15:06:17.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:17.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:17.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:17.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:17.825574 14662 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:17.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:17.834122 14662 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:17.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:17.834975 14662 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:17.907182 13311 debug-util.cc:398] Leaking SignalData structure 0x7b08000a1020 after lost signal to thread 13243
15:06:17.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:17.908732 13311 debug-util.cc:398] Leaking SignalData structure 0x7b08000bde80 after lost signal to thread 13314
15:06:18.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.242208 14766 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.123.126:46369 every 8 connection(s)
15:06:18.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.242990 14662 rpc_server.cc:307] RPC server started. Bound to: 127.13.123.126:46369
15:06:18.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.246888 14662 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-0/data/info.pb
15:06:18.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.254415 13805 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu as pid 14662
15:06:18.256 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:18.256 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.13.123.124:43073
15:06:18.256 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.258682 13805 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/wal
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/data
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/logs
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/data/info.pb
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.123.124:43073
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.123.124
15:06:18.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36561
15:06:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.124.keytab
15:06:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.123.124
15:06:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:06:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:06:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.123.84:33031
15:06:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.123.126:46369,127.13.123.125:44729,127.13.123.124:43073
15:06:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42691/jwks.json
15:06:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:06:18.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.273483 14767 sys_catalog.cc:263] Verifying existing consensus state
15:06:18.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.281747 14767 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994: Bootstrap starting.
15:06:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.335949 14767 log.cc:826] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994: Log is configured to *not* fsync() on all Append() calls
15:06:18.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.362937 14767 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:06:18.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.364012 14767 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994: Bootstrap complete.
15:06:18.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.404649 14767 raft_consensus.cc:357] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } }
15:06:18.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.408638 14767 raft_consensus.cc:738] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: fbe595fcd2124f2b9c575098f0832994, State: Initialized, Role: FOLLOWER
15:06:18.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.409888 14767 consensus_queue.cc:260] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } }
15:06:18.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.418542 14771 sys_catalog.cc:455] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } } }
15:06:18.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.419471 14771 sys_catalog.cc:458] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [sys.catalog]: This master's current role is: FOLLOWER
15:06:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.439877 14767 sys_catalog.cc:564] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [sys.catalog]: configured and running, proceeding with master startup.
15:06:18.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:18.460702 14774 negotiation.cc:337] Failed RPC negotiation. Trace:
15:06:18.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:18.424262 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.13.123.65:39319 (local address 127.13.123.126:46369)
15:06:18.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:18.448861 (+ 24599us) server_negotiation.cc:206] Beginning negotiation
15:06:18.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:18.448909 (+ 48us) server_negotiation.cc:399] Waiting for connection header
15:06:18.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:18.448994 (+ 85us) server_negotiation.cc:407] Connection header received
15:06:18.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:18.449627 (+ 633us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
15:06:18.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:18.449672 (+ 45us) server_negotiation.cc:461] Received NEGOTIATE request from client
15:06:18.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:18.450144 (+ 472us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
15:06:18.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:18.450553 (+ 409us) server_negotiation.cc:226] Negotiated authn=SASL
15:06:18.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:18.458732 (+ 8179us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.13.123.65:39319: BlockingRecv error: recv got EOF from 127.13.123.65:39319 (error 108)
15:06:18.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":23685,"thread_start_us":26655,"threads_started":1}
15:06:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.525270 14786 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:06:18.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:18 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521577, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.126@KRBTEST.COM for kudu/127.13.123.126@KRBTEST.COM
15:06:18.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.593478 14423 heartbeater.cc:346] Connected to a master server at 127.13.123.126:46369
15:06:18.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.602454 14732 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "82402bf20ce84da08e47cd79295ab879" instance_seqno: 1736521571314526) as {username='kudu', principal='kudu/127.13.123.66@KRBTEST.COM'} at 127.13.123.66:39997; Asking this server to re-register.
15:06:18.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.619055 14423 heartbeater.cc:463] Registering TS with master...
15:06:18.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.622881 14731 ts_manager.cc:194] Registered new tserver with Master: 82402bf20ce84da08e47cd79295ab879 (127.13.123.66:40567)
15:06:18.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:18 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521577, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.126@KRBTEST.COM for kudu/127.13.123.125@KRBTEST.COM
15:06:18.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.673377 14570 heartbeater.cc:346] Connected to a master server at 127.13.123.126:46369
15:06:18.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.676553 14732 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "5c08693f59ea487391477364ecf7f0f5" instance_seqno: 1736521573399616) as {username='kudu', principal='kudu/127.13.123.67@KRBTEST.COM'} at 127.13.123.67:54869; Asking this server to re-register.
15:06:18.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.678194 14570 heartbeater.cc:463] Registering TS with master...
15:06:18.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.681340 14732 ts_manager.cc:194] Registered new tserver with Master: 5c08693f59ea487391477364ecf7f0f5 (127.13.123.67:40771)
15:06:18.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.708869 14662 master_runner.cc:186] Error getting master registration for 127.13.123.124:43073: Network error: Client connection negotiation failed: client connection to 127.13.123.124:43073: connect: Connection refused (error 111),
15:06:18.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:18 dist-test-slave-6prw krb5kdc[13817](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521578, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:18.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.795941 14662 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:06:18.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:18.796732 14160 init.cc:295] Successfully reacquired a new kerberos TGT
15:06:19.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.261161 14797 raft_consensus.cc:491] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:06:19.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.262038 14797 raft_consensus.cc:513] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } }
15:06:19.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:19.275005 14769 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:19.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:19.275843 14769 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:19.276996 14769 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:19.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.283763 14797 leader_election.cc:290] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers fbe595fcd2124f2b9c575098f0832994 (127.13.123.126:46369), e698f9702ebe4cbdb156b67d1d5ad325 (127.13.123.124:43073)
15:06:19.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:19.313764 14611 leader_election.cc:336] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer e698f9702ebe4cbdb156b67d1d5ad325 (127.13.123.124:43073): Network error: Client connection negotiation failed: client connection to 127.13.123.124:43073: connect: Connection refused (error 111)
15:06:19.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:19 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521575, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.125@KRBTEST.COM for kudu/127.13.123.126@KRBTEST.COM
15:06:19.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.365407 14742 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "fbe595fcd2124f2b9c575098f0832994" is_pre_election: true
15:06:19.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.366361 14742 raft_consensus.cc:2463] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4c0b2ca5dec748b292bc8fc2a05e02e0 in term 1.
15:06:19.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.368130 14610 leader_election.cc:304] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4c0b2ca5dec748b292bc8fc2a05e02e0, fbe595fcd2124f2b9c575098f0832994; no voters: e698f9702ebe4cbdb156b67d1d5ad325
15:06:19.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.369248 14797 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [term 1 FOLLOWER]: Leader pre-election won for term 2
15:06:19.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.369580 14797 raft_consensus.cc:491] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:06:19.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.369879 14797 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [term 1 FOLLOWER]: Advancing to term 2
15:06:19.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:19.377421 14769 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:06:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:19.377831 14769 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:06:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:19.378494 14769 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:19.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:19.379009 14769 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:06:19.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:19.379388 14769 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:06:19.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:19.379760 14769 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:06:19.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:19.380041 14769 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42691/jwks.json
15:06:19.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.381708 14797 raft_consensus.cc:513] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } }
15:06:19.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.394515 14742 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "fbe595fcd2124f2b9c575098f0832994"
15:06:19.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.395144 14742 raft_consensus.cc:3054] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [term 1 FOLLOWER]: Advancing to term 2
15:06:19.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.404774 14797 leader_election.cc:290] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [CANDIDATE]: Term 2 election: Requested vote from peers fbe595fcd2124f2b9c575098f0832994 (127.13.123.126:46369), e698f9702ebe4cbdb156b67d1d5ad325 (127.13.123.124:43073)
15:06:19.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.405985 14742 raft_consensus.cc:2463] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4c0b2ca5dec748b292bc8fc2a05e02e0 in term 2.
15:06:19.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.407433 14610 leader_election.cc:304] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4c0b2ca5dec748b292bc8fc2a05e02e0, fbe595fcd2124f2b9c575098f0832994; no voters:
15:06:19.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.408459 14797 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [term 2 FOLLOWER]: Leader election won for term 2
15:06:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:19.408955 14611 leader_election.cc:336] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer e698f9702ebe4cbdb156b67d1d5ad325 (127.13.123.124:43073): Network error: Client connection negotiation failed: client connection to 127.13.123.124:43073: connect: Connection refused (error 111)
15:06:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.413146 14797 raft_consensus.cc:695] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [term 2 LEADER]: Becoming Leader. State: Replica: 4c0b2ca5dec748b292bc8fc2a05e02e0, State: Running, Role: LEADER
15:06:19.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.414383 14797 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } }
15:06:19.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.445871 14803 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4c0b2ca5dec748b292bc8fc2a05e02e0. Latest consensus state: current_term: 2 leader_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } } }
15:06:19.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.447170 14803 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [sys.catalog]: This master's current role is: LEADER
15:06:19.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.452569 14806 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
15:06:19.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.468740 14806 catalog_manager.cc:1485] Initializing Kudu cluster ID...
15:06:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.470916 14769 master_runner.cc:386] Master server non-default flags:
15:06:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.123.84:33031
15:06:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/data
15:06:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/wal
15:06:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:06:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.123.126:46369,127.13.123.125:44729,127.13.123.124:43073
15:06:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/krb5kdc/kudu_127.13.123.124.keytab
15:06:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.123.124
15:06:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.123.124:43073
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42691/jwks.json
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/data/info.pb
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.123.124
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36561
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/logs
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 14:43:22 UTC on a633016ff5c9
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4112
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:19 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521578, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.65@KRBTEST.COM for kudu/127.13.123.126@KRBTEST.COM
15:06:19.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.472723 14769 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:19.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.474279 14769 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:19.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.489703 14806 catalog_manager.cc:1260] Loaded cluster ID: 471e1f0fb6504d9fafc5812ca92526b3
15:06:19.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.490255 14806 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
15:06:19.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.495636 14806 catalog_manager.cc:1505] Loading token signing keys...
15:06:19.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.498874 14279 heartbeater.cc:346] Connected to a master server at 127.13.123.126:46369
15:06:19.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.500391 14806 catalog_manager.cc:5910] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0: Loaded TSK: 0
15:06:19.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.501719 14806 catalog_manager.cc:1515] Initializing in-progress tserver states...
15:06:19.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.502290 14732 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "342c965824f04206b4a5938b066b652f" instance_seqno: 1736521569176882) as {username='kudu', principal='kudu/127.13.123.65@KRBTEST.COM'} at 127.13.123.65:60795; Asking this server to re-register.
15:06:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.503904 14279 heartbeater.cc:463] Registering TS with master...
15:06:19.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.507294 14732 ts_manager.cc:194] Registered new tserver with Master: 342c965824f04206b4a5938b066b652f (127.13.123.65:42801)
15:06:19.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:19.517750 14808 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:19.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.560804 14786 catalog_manager.cc:1260] Loaded cluster ID: 471e1f0fb6504d9fafc5812ca92526b3
15:06:19.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.561084 14786 catalog_manager.cc:1553] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994: loading cluster ID for follower catalog manager: success
15:06:19.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.598835 14786 catalog_manager.cc:1575] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994: acquiring CA information for follower catalog manager: success
15:06:19.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:19.621287 14786 catalog_manager.cc:1603] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:06:19.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:19.970469 14611 consensus_peers.cc:487] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 -> Peer e698f9702ebe4cbdb156b67d1d5ad325 (127.13.123.124:43073): Couldn't send request to peer e698f9702ebe4cbdb156b67d1d5ad325. Status: Network error: Client connection negotiation failed: client connection to 127.13.123.124:43073: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
15:06:20.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:20.002620 14742 raft_consensus.cc:1270] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [term 2 FOLLOWER]: Refusing update from remote peer 4c0b2ca5dec748b292bc8fc2a05e02e0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
15:06:20.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:20.004505 14803 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
15:06:20.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:20.058425 14821 sys_catalog.cc:455] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4c0b2ca5dec748b292bc8fc2a05e02e0. Latest consensus state: current_term: 2 leader_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } } }
15:06:20.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:20.059303 14821 sys_catalog.cc:458] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [sys.catalog]: This master's current role is: FOLLOWER
15:06:20.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:20.065145 14803 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } } }
15:06:20.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:20.066093 14803 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [sys.catalog]: This master's current role is: LEADER
15:06:20.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:20.073587 14821 sys_catalog.cc:455] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } } }
15:06:20.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:20.074404 14821 sys_catalog.cc:458] T 00000000000000000000000000000000 P fbe595fcd2124f2b9c575098f0832994 [sys.catalog]: This master's current role is: FOLLOWER
15:06:20.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:20.550208 14421 heartbeater.cc:502] Master 127.13.123.125:44729 was elected leader, sending a full tablet report...
15:06:20.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:20.649317 14566 heartbeater.cc:502] Master 127.13.123.125:44729 was elected leader, sending a full tablet report...
15:06:20.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:20.888343 14581 master_runner.cc:186] Error getting master registration for 127.13.123.124:43073: Network error: Client connection negotiation failed: client connection to 127.13.123.124:43073: connect: Connection refused (error 111),
15:06:20.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:20.888785 14581 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:06:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:21 dist-test-slave-6prw krb5kdc[13817](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521581, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:21.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:21.105836 14304 init.cc:295] Successfully reacquired a new kerberos TGT
15:06:21.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:21.456996 14277 heartbeater.cc:502] Master 127.13.123.125:44729 was elected leader, sending a full tablet report...
15:06:22.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:22.282280 14611 consensus_peers.cc:487] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 -> Peer e698f9702ebe4cbdb156b67d1d5ad325 (127.13.123.124:43073): Couldn't send request to peer e698f9702ebe4cbdb156b67d1d5ad325. Status: Network error: Client connection negotiation failed: client connection to 127.13.123.124:43073: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
15:06:22.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:20.896661 14807 debug-util.cc:398] Leaking SignalData structure 0x7b080001cac0 after lost signal to thread 14769
15:06:22.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:22.341921 14807 kernel_stack_watchdog.cc:198] Thread 14769 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:636 for 371ms:
15:06:22.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:06:22.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:06:22.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:06:22.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:06:22.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:22.342604 14769 thread.cc:635] OpenStack (cloud detector) Time spent creating pthread: real 2.819s user 0.483s sys 0.907s
15:06:22.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:19.532811 14809 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:22.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:22.343039 14769 thread.cc:601] OpenStack (cloud detector) Time spent starting thread: real 2.820s user 0.483s sys 0.907s
15:06:22.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:22.345155 14811 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:22.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:22.355370 14810 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2803 milliseconds
15:06:22.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.355446 14769 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:06:22.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.357162 14769 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:22.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.360162 14769 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:22.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.361629 14769 hybrid_clock.cc:648] HybridClock initialized: now 1736521582361522 us; error 111 us; skew 500 ppm
15:06:22.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.362638 14769 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:22.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:22 dist-test-slave-6prw krb5kdc[13817](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521582, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:22.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.369297 14769 init.cc:377] Logged in from keytab as kudu/127.13.123.124@KRBTEST.COM (short username kudu)
15:06:22.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.374934 14769 webserver.cc:458] Webserver started at http://127.13.123.124:36561/ using document root <none> and password file <none>
15:06:22.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.375849 14769 fs_manager.cc:362] Metadata directory not provided
15:06:22.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.376088 14769 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:22.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.384334 14769 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.007s sys 0.001s
15:06:22.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.388732 14844 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:22.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.389854 14769 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.002s
15:06:22.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.390177 14769 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/data,/tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/wal
15:06:22.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e698f9702ebe4cbdb156b67d1d5ad325"
15:06:22.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 15:06:03 on dist-test-slave-6prw"
15:06:22.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.392668 14769 fs_report.cc:389] FS layout report
15:06:22.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:22.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/wal
15:06:22.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/wal
15:06:22.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/data/data
15:06:22.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:22.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:22.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:22.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:22.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:22.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:22.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:22.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:22.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:22.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:22.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:22.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.466958 14769 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:22.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.472725 14769 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:22.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.473173 14769 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:22.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.579303 14769 rpc_server.cc:307] RPC server started. Bound to: 127.13.123.124:43073
15:06:22.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.579464 14896 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.123.124:43073 every 8 connection(s)
15:06:22.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.583572 14769 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskyZ3UZe/test-tmp/mini-kudu-cluster4597038286385078585/master-2/data/info.pb
15:06:22.586 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.585377 13805 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu as pid 14769
15:06:22.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.613093 14897 sys_catalog.cc:263] Verifying existing consensus state
15:06:22.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.625100 14897 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325: Bootstrap starting.
15:06:22.696 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.13.123.125:44729(127.13.123.125:44729)] connecting to peer
15:06:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:22 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521575, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.125@KRBTEST.COM for kudu/127.13.123.124@KRBTEST.COM
15:06:22.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:22 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521581, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.66@KRBTEST.COM for kudu/127.13.123.124@KRBTEST.COM
15:06:22.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.753429 14422 heartbeater.cc:346] Connected to a master server at 127.13.123.124:43073
15:06:22.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.793733 14567 heartbeater.cc:346] Connected to a master server at 127.13.123.124:43073
15:06:22.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.794272 14567 heartbeater.cc:502] Master 127.13.123.124:43073 was elected leader, sending a full tablet report...
15:06:22.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.822288 14897 log.cc:826] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325: Log is configured to *not* fsync() on all Append() calls
15:06:22.822 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.13.123.126:46369(127.13.123.126:46369)] connecting to peer
15:06:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.852535 14897 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:06:22.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.853523 14897 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325: Bootstrap complete.
15:06:22.865 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.13.123.124:43073(127.13.123.124:43073)] connecting to peer
15:06:22.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.892340 14897 raft_consensus.cc:357] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } }
15:06:22.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.895752 14897 raft_consensus.cc:738] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: e698f9702ebe4cbdb156b67d1d5ad325, State: Initialized, Role: FOLLOWER
15:06:22.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.896916 14897 consensus_queue.cc:260] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } }
15:06:22.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.905385 14908 sys_catalog.cc:455] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } } }
15:06:22.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.906203 14908 sys_catalog.cc:458] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [sys.catalog]: This master's current role is: FOLLOWER
15:06:22.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.909780 14897 sys_catalog.cc:564] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [sys.catalog]: configured and running, proceeding with master startup.
15:06:22.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:22.911613 14872 raft_consensus.cc:3054] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [term 1 FOLLOWER]: Advancing to term 2
15:06:22.952 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.13.123.126:46369(127.13.123.126:46369)] Successfully connected to peer
15:06:22.953 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.13.123.125:44729(127.13.123.125:44729)] Successfully connected to peer
15:06:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:23.021747 14918 sys_catalog.cc:455] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } } }
15:06:23.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:23.023751 14918 sys_catalog.cc:458] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [sys.catalog]: This master's current role is: FOLLOWER
15:06:23.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:23.027107 14908 sys_catalog.cc:455] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4c0b2ca5dec748b292bc8fc2a05e02e0. Latest consensus state: current_term: 2 leader_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } } }
15:06:23.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:23.028095 14908 sys_catalog.cc:458] T 00000000000000000000000000000000 P e698f9702ebe4cbdb156b67d1d5ad325 [sys.catalog]: This master's current role is: FOLLOWER
15:06:23.055 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.13.123.124:43073(127.13.123.124:43073)] Successfully connected to peer
15:06:23.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:23.057938 14924 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:06:23.082 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
15:06:23.088 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.123.125:44729(127.13.123.125:44729)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
15:06:23.088 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.123.126:46369(127.13.123.126:46369)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
15:06:23.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:23 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521582, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.124@KRBTEST.COM for kudu/127.13.123.126@KRBTEST.COM
15:06:23.091 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.123.126:46369: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
15:06:23.091 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.123.125:44729: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
15:06:23.093 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.123.124:43073(127.13.123.124:43073)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
15:06:23.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:23.093966 14911 negotiation.cc:337] Failed RPC negotiation. Trace:
15:06:23.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:22.947363 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:46880 (local address 127.13.123.125:44729)
15:06:23.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:22.951881 (+ 4518us) server_negotiation.cc:206] Beginning negotiation
15:06:23.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:22.951906 (+ 25us) server_negotiation.cc:399] Waiting for connection header
15:06:23.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:23.019625 (+ 67719us) server_negotiation.cc:407] Connection header received
15:06:23.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:23.054876 (+ 35251us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
15:06:23.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:23.054924 (+ 48us) server_negotiation.cc:461] Received NEGOTIATE request from client
15:06:23.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:23.055251 (+ 327us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
15:06:23.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:23.055488 (+ 237us) server_negotiation.cc:226] Negotiated authn=SASL
15:06:23.094 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.13.123.126:46369(127.13.123.126:46369)] handling channelInactive
15:06:23.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:23.092515 (+ 37027us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:46880: BlockingRecv error: recv got EOF from 127.0.0.1:46880 (error 108)
15:06:23.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":3926,"thread_start_us":3616,"threads_started":1}
15:06:23.095 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.123.124:43073: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
15:06:23.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:23.096899 14910 negotiation.cc:337] Failed RPC negotiation. Trace:
15:06:23.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:22.947232 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:57640 (local address 127.13.123.126:46369)
15:06:23.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:22.977585 (+ 30353us) server_negotiation.cc:206] Beginning negotiation
15:06:23.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:22.977603 (+ 18us) server_negotiation.cc:399] Waiting for connection header
15:06:23.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:23.030248 (+ 52645us) server_negotiation.cc:407] Connection header received
15:06:23.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:23.055914 (+ 25666us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
15:06:23.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:23.055943 (+ 29us) server_negotiation.cc:461] Received NEGOTIATE request from client
15:06:23.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:23.056149 (+ 206us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
15:06:23.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:23.056342 (+ 193us) server_negotiation.cc:226] Negotiated authn=SASL
15:06:23.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:23.095576 (+ 39234us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:57640: BlockingRecv error: recv got EOF from 127.0.0.1:57640 (error 108)
15:06:23.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":29936,"thread_start_us":29654,"threads_started":1}
15:06:23.101 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=510), Traces: [0ms] refreshing cache from master, [75ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.123.125:44729, [214ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.123.126:46369, [258ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.123.124:43073, [483ms] Sub RPC ConnectToMaster: received response from server master-127.13.123.125:44729: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [483ms] Sub RPC ConnectToMaster: received response from server master-127.13.123.126:46369: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [486ms] Sub RPC ConnectToMaster: received response from server master-127.13.123.124:43073: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.13.123.125:44729,127.13.123.126:46369,127.13.123.124:43073). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) [main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) [main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) [async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) [main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) [main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) [main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) [netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.13.123.125:44729,127.13.123.126:46369,127.13.123.124:43073). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
15:06:23.120 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=529), Traces: [0ms] refreshing cache from master, [75ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.123.125:44729, [214ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.123.126:46369, [258ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.123.124:43073, [483ms] Sub RPC ConnectToMaster: received response from server master-127.13.123.125:44729: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [483ms] Sub RPC ConnectToMaster: received response from server master-127.13.123.126:46369: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [486ms] Sub RPC ConnectToMaster: received response from server master-127.13.123.124:43073: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) with no deferred attached!
15:06:23.125 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.13.123.126:46369(127.13.123.126:46369)] disconnecting while in state TERMINATED
15:06:23.126 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.13.123.124:43073(127.13.123.124:43073)] disconnecting while in state TERMINATED
15:06:23.126 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.13.123.125:44729(127.13.123.125:44729)] disconnecting while in state TERMINATED
15:06:23.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 15:06:23.139729 14904 negotiation.cc:337] Failed RPC negotiation. Trace:
15:06:23.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:22.952285 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:47644 (local address 127.13.123.124:43073)
15:06:23.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:22.952949 (+ 664us) server_negotiation.cc:206] Beginning negotiation
15:06:23.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:22.952974 (+ 25us) server_negotiation.cc:399] Waiting for connection header
15:06:23.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:23.055686 (+102712us) server_negotiation.cc:407] Connection header received
15:06:23.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:23.056653 (+ 967us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
15:06:23.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:23.056684 (+ 31us) server_negotiation.cc:461] Received NEGOTIATE request from client
15:06:23.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:23.056914 (+ 230us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
15:06:23.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:23.057075 (+ 161us) server_negotiation.cc:226] Negotiated authn=SASL
15:06:23.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 15:06:23.138133 (+ 81058us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:47644: BlockingRecv error: recv got EOF from 127.0.0.1:47644 (error 108)
15:06:23.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":97}
15:06:23.142 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.13.123.125:44729(127.13.123.125:44729)] handling channelInactive
15:06:23.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:23 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521582, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.124@KRBTEST.COM for kudu/127.13.123.125@KRBTEST.COM
15:06:23.143 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:06:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:23.145364 14803 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbe595fcd2124f2b9c575098f0832994" member_type: VOTER last_known_addr { host: "127.13.123.126" port: 46369 } } peers { permanent_uuid: "4c0b2ca5dec748b292bc8fc2a05e02e0" member_type: VOTER last_known_addr { host: "127.13.123.125" port: 44729 } } peers { permanent_uuid: "e698f9702ebe4cbdb156b67d1d5ad325" member_type: VOTER last_known_addr { host: "127.13.123.124" port: 43073 } } }
15:06:23.157 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.13.123.124:43073(127.13.123.124:43073)] handling channelInactive
15:06:23.157 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:06:23.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:23.152372 14803 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4c0b2ca5dec748b292bc8fc2a05e02e0 [sys.catalog]: This master's current role is: LEADER
15:06:23.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:23.168277 13805 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu with pid 14123
15:06:23.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:23 dist-test-slave-6prw krb5kdc[13817](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521583, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:23.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 15:06:23 dist-test-slave-6prw krb5kdc[13817](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736521582, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.123.124@KRBTEST.COM for kudu/127.13.123.124@KRBTEST.COM
15:06:23.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:23.175944 14449 init.cc:295] Successfully reacquired a new kerberos TGT
15:06:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:23.201644 13805 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu with pid 14289
15:06:23.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:23.234689 13805 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu with pid 14433
15:06:23.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:23.268260 13805 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu with pid 14662
15:06:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:23.297219 13805 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu with pid 14581
15:06:23.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:23.326205 13805 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu with pid 14769
15:06:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-10T15:06:24Z chronyd exiting
15:06:25.548 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:06:25.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:25.550146 13081 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu with pid 13361
15:06:25.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:25.576560 13081 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu with pid 13517
15:06:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:25.602967 13081 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu with pid 13660
15:06:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:25.632366 13081 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu with pid 13093
15:06:25.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:25.661568 13081 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu with pid 13163
15:06:25.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 15:06:25.692873 13081 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskyZ3UZe/build/tsan/bin/kudu with pid 13238
15:06:26.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-10T15:06:26Z chronyd exiting
15:06:27.866 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.13.123.125:44729,127.13.123.126:46369,127.13.123.124:43073). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.13.123.125:44729,127.13.123.126:46369,127.13.123.124:43073). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more