10:27:07.018 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
10:27:07.018 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskaxjw05/test-tmp
10:27:07.018 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
10:27:07.019 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskaxjw05/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
10:27:07.020 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
10:27:07.339 [DEBUG - main] (MiniKuduCluster.java:195) Response:
10:27:07.339 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
10:27:07.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:07.342061 24301 mini_oidc.cc:150] Starting JWKS server
10:27:07.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:07.345595 24301 webserver.cc:458] Webserver started at http://127.0.0.1:43225/ using document root <none> and password file <none>
10:27:07.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:07.346007 24301 mini_oidc.cc:186] Starting OIDC Discovery server
10:27:07.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:07.347681 24301 webserver.cc:458] Webserver started at http://127.0.0.1:32819/ using document root <none> and password file <none>
10:27:07.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-15T10:27:07Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
10:27:07.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-15T10:27:07Z Disabled control of system clock
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:07.411798 24301 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-0/wal
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-0/data
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-0/logs
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-0/data/info.pb
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.187.126:41049
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.187.126
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.187.84:44865
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.187.126:41049,127.23.187.125:45989,127.23.187.124:38041
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43225/jwks.json with env {}
10:27:07.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:07.700826 24313 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:07.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:07.701390 24313 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:07.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:07.701871 24313 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:07.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:07.731758 24313 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:27:07.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:07.732030 24313 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:07.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:07.732226 24313 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:27:07.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:07.732419 24313 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:27:07.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:07.732597 24313 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:27:07.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:07.732734 24313 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43225/jwks.json
10:27:07.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:07.766438 24313 master_runner.cc:386] Master server non-default flags:
10:27:07.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:07.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.187.84:44865
10:27:07.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:07.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:07.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-0/data
10:27:07.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-0/wal
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.187.126:41049,127.23.187.125:45989,127.23.187.124:38041
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.187.126:41049
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43225/jwks.json
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-0/data/info.pb
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.187.126
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-0/logs
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:07.767716 24313 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:07.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:07.769306 24313 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:27:07.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:07.783869 24319 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:08.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:07.784020 24320 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:08.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:07.786569 24313 server_base.cc:1034] running on GCE node
10:27:08.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:07.786299 24322 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:08.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:08.949394 24313 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:08.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:08.952478 24313 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:08.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:08.953948 24313 hybrid_clock.cc:648] HybridClock initialized: now 1736936828953906 us; error 59 us; skew 500 ppm
10:27:08.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:08.954746 24313 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:08.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:08.961587 24313 webserver.cc:458] Webserver started at http://127.23.187.126:46171/ using document root <none> and password file <none>
10:27:08.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:08.962493 24313 fs_manager.cc:362] Metadata directory not provided
10:27:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:08.962678 24313 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:08.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:08.963035 24313 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:27:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:08.967650 24313 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-0/data/instance:
10:27:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3390f8eca7534b5ab3700276aa09044e"
10:27:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:08 on dist-test-slave-hht6"
10:27:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:08.968686 24313 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-0/wal/instance:
10:27:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3390f8eca7534b5ab3700276aa09044e"
10:27:08.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:08 on dist-test-slave-hht6"
10:27:08.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:08.975648 24313 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.001s sys 0.004s
10:27:08.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:08.981160 24329 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:08.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:08.982157 24313 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
10:27:08.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:08.982425 24313 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-0/data,/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-0/wal
10:27:08.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3390f8eca7534b5ab3700276aa09044e"
10:27:08.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:08 on dist-test-slave-hht6"
10:27:08.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:08.982728 24313 fs_report.cc:389] FS layout report
10:27:08.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:08.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-0/wal
10:27:08.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-0/wal
10:27:08.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-0/data/data
10:27:08.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:08.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:08.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:08.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:08.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:08.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:08.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:08.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:08.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:08.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:08.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:09.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:09.045625 24313 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:27:09.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:09.047060 24313 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:09.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:09.047459 24313 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:09.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:09.115924 24313 rpc_server.cc:307] RPC server started. Bound to: 127.23.187.126:41049
10:27:09.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:09.116004 24380 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.187.126:41049 every 8 connection(s)
10:27:09.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:09.118629 24313 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-0/data/info.pb
10:27:09.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:09.123571 24381 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:27:09.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:09.129523 24301 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu as pid 24313
10:27:09.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:09.130079 24301 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-0/wal/instance
10:27:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:09.135455 24301 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-1/wal
10:27:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-1/data
10:27:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:09.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-1/logs
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-1/data/info.pb
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.187.125:45989
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.187.125
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.187.84:44865
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.187.126:41049,127.23.187.125:45989,127.23.187.124:38041
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43225/jwks.json with env {}
10:27:09.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:09.137825 24381 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.187.126" port: 41049 } has no permanent_uuid. Determining permanent_uuid...
10:27:09.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:09.163057 24381 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } has no permanent_uuid. Determining permanent_uuid...
10:27:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:09.165345 24332 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.187.125:45989: connect: Connection refused (error 111)
10:27:09.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:09.169507 24381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.187.125:45989: Network error: Client connection negotiation failed: client connection to 127.23.187.125:45989: connect: Connection refused (error 111)
10:27:09.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:09.219048 24381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } attempt: 1
10:27:09.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:09.222754 24381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.187.125:45989: Network error: Client connection negotiation failed: client connection to 127.23.187.125:45989: connect: Connection refused (error 111)
10:27:09.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:09.291296 24381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } attempt: 2
10:27:09.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:09.295233 24381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.187.125:45989: Network error: Client connection negotiation failed: client connection to 127.23.187.125:45989: connect: Connection refused (error 111)
10:27:09.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:09.386721 24381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } attempt: 3
10:27:09.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:09.390305 24381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.187.125:45989: Network error: Client connection negotiation failed: client connection to 127.23.187.125:45989: connect: Connection refused (error 111)
10:27:09.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:09.436866 24383 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:09.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:09.437469 24383 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:09.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:09.437985 24383 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:09.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:09.467079 24383 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:27:09.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:09.467351 24383 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:09.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:09.467577 24383 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:27:09.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:09.467769 24383 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:27:09.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:09.467952 24383 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:27:09.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:09.468133 24383 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43225/jwks.json
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:09.500922 24383 master_runner.cc:386] Master server non-default flags:
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.187.84:44865
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-1/data
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-1/wal
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.187.126:41049,127.23.187.125:45989,127.23.187.124:38041
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.187.125:45989
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43225/jwks.json
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-1/data/info.pb
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.187.125
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-1/logs
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:27:09.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:09.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:27:09.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:09.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:27:09.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:27:09.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:09.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:09.502187 24383 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:09.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:09.503563 24383 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:27:09.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:09.518945 24392 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:09.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:09.533789 24381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } attempt: 4
10:27:09.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:09.537436 24381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.187.125:45989: Network error: Client connection negotiation failed: client connection to 127.23.187.125:45989: connect: Connection refused (error 111)
10:27:09.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:09.837072 24381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } attempt: 5
10:27:09.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:09.842583 24381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.187.125:45989: Network error: Client connection negotiation failed: client connection to 127.23.187.125:45989: connect: Connection refused (error 111)
10:27:10.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.390024 24381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } attempt: 6
10:27:10.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:10.399468 24381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.187.125:45989: Network error: Client connection negotiation failed: client connection to 127.23.187.125:45989: connect: Connection refused (error 111)
10:27:10.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:09.519177 24395 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:10.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:09.519434 24393 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:10.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:09.520846 24383 server_base.cc:1034] running on GCE node
10:27:10.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.705864 24383 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.708473 24383 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:10.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.709825 24383 hybrid_clock.cc:648] HybridClock initialized: now 1736936830709782 us; error 55 us; skew 500 ppm
10:27:10.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.710633 24383 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.720337 24383 webserver.cc:458] Webserver started at http://127.23.187.125:40005/ using document root <none> and password file <none>
10:27:10.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.721308 24383 fs_manager.cc:362] Metadata directory not provided
10:27:10.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.721509 24383 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:10.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.721899 24383 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:27:10.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.726720 24383 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-1/data/instance:
10:27:10.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d4ecff1756e24efbb08a49f672be43a8"
10:27:10.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:10 on dist-test-slave-hht6"
10:27:10.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.727754 24383 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-1/wal/instance:
10:27:10.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d4ecff1756e24efbb08a49f672be43a8"
10:27:10.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:10 on dist-test-slave-hht6"
10:27:10.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.734874 24383 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.003s
10:27:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.740182 24404 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.741168 24383 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
10:27:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.741506 24383 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-1/data,/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-1/wal
10:27:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d4ecff1756e24efbb08a49f672be43a8"
10:27:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:10 on dist-test-slave-hht6"
10:27:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.741833 24383 fs_report.cc:389] FS layout report
10:27:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-1/wal
10:27:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-1/wal
10:27:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-1/data/data
10:27:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:10.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.789299 24383 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:27:10.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.790772 24383 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:10.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.791298 24383 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:10.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.862263 24383 rpc_server.cc:307] RPC server started. Bound to: 127.23.187.125:45989
10:27:10.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.862339 24455 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.187.125:45989 every 8 connection(s)
10:27:10.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.864861 24383 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-1/data/info.pb
10:27:10.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.868219 24301 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu as pid 24383
10:27:10.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.868709 24301 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-1/wal/instance
10:27:10.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.870913 24456 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.871774 24301 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-2/wal
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-2/data
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-2/logs
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-2/data/info.pb
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.187.124:38041
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.187.124
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.187.84:44865
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.187.126:41049,127.23.187.125:45989,127.23.187.124:38041
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:10.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43225/jwks.json with env {}
10:27:10.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.884618 24456 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.187.126" port: 41049 } has no permanent_uuid. Determining permanent_uuid...
10:27:10.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.906458 24456 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } has no permanent_uuid. Determining permanent_uuid...
10:27:10.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.920722 24456 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } has no permanent_uuid. Determining permanent_uuid...
10:27:10.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:10.923163 24408 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.187.124:38041: connect: Connection refused (error 111)
10:27:10.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:10.927285 24456 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.187.124:38041: Network error: Client connection negotiation failed: client connection to 127.23.187.124:38041: connect: Connection refused (error 111)
10:27:10.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:10.976734 24456 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } attempt: 1
10:27:10.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:10.980805 24456 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.187.124:38041: Network error: Client connection negotiation failed: client connection to 127.23.187.124:38041: connect: Connection refused (error 111)
10:27:11.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:11.049182 24456 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } attempt: 2
10:27:11.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:11.053084 24456 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.187.124:38041: Network error: Client connection negotiation failed: client connection to 127.23.187.124:38041: connect: Connection refused (error 111)
10:27:11.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:11.144567 24456 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } attempt: 3
10:27:11.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:11.148527 24456 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.187.124:38041: Network error: Client connection negotiation failed: client connection to 127.23.187.124:38041: connect: Connection refused (error 111)
10:27:11.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:11.184489 24458 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:11.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:11.185070 24458 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:11.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:11.185551 24458 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:11.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:11.216475 24458 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:27:11.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:11.216823 24458 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:11.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:11.217126 24458 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:27:11.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:11.217331 24458 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:27:11.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:11.217528 24458 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:27:11.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:11.217672 24458 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43225/jwks.json
10:27:11.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:11.252391 24458 master_runner.cc:386] Master server non-default flags:
10:27:11.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:11.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.187.84:44865
10:27:11.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:11.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:11.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-2/data
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-2/wal
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.187.126:41049,127.23.187.125:45989,127.23.187.124:38041
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.187.124:38041
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43225/jwks.json
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-2/data/info.pb
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.187.124
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-2/logs
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:11.253722 24458 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:11.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:11.255175 24458 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:27:11.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:11.270855 24471 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:11.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:11.292038 24456 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } attempt: 4
10:27:11.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:11.296993 24456 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.187.124:38041: Network error: Client connection negotiation failed: client connection to 127.23.187.124:38041: connect: Connection refused (error 111)
10:27:11.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:11.460059 24381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } attempt: 7
10:27:11.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:11.475580 24381 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } has no permanent_uuid. Determining permanent_uuid...
10:27:11.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:11.481669 24381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.187.124:38041: Network error: Client connection negotiation failed: client connection to 127.23.187.124:38041: connect: Connection refused (error 111)
10:27:11.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:11.540010 24381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } attempt: 1
10:27:11.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:11.544060 24381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.187.124:38041: Network error: Client connection negotiation failed: client connection to 127.23.187.124:38041: connect: Connection refused (error 111)
10:27:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:11.596376 24456 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } attempt: 5
10:27:11.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:11.601253 24456 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.187.124:38041: Network error: Client connection negotiation failed: client connection to 127.23.187.124:38041: connect: Connection refused (error 111)
10:27:11.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:11.625656 24381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } attempt: 2
10:27:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:11.629878 24381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.187.124:38041: Network error: Client connection negotiation failed: client connection to 127.23.187.124:38041: connect: Connection refused (error 111)
10:27:11.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:11.715551 24381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } attempt: 3
10:27:11.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:11.720351 24381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.187.124:38041: Network error: Client connection negotiation failed: client connection to 127.23.187.124:38041: connect: Connection refused (error 111)
10:27:11.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:11.860783 24381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } attempt: 4
10:27:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:11.864787 24381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.187.124:38041: Network error: Client connection negotiation failed: client connection to 127.23.187.124:38041: connect: Connection refused (error 111)
10:27:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.148313 24381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } attempt: 5
10:27:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.148653 24456 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } attempt: 6
10:27:12.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:12.153276 24381 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.187.124:38041: Network error: Client connection negotiation failed: client connection to 127.23.187.124:38041: connect: Connection refused (error 111)
10:27:12.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:12.156980 24456 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.187.124:38041: Network error: Client connection negotiation failed: client connection to 127.23.187.124:38041: connect: Connection refused (error 111)
10:27:12.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:11.271879 24469 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:12.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:11.272725 24458 server_base.cc:1034] running on GCE node
10:27:12.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:11.270869 24468 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.465569 24458 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:12.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.468154 24458 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:12.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.469516 24458 hybrid_clock.cc:648] HybridClock initialized: now 1736936832469464 us; error 70 us; skew 500 ppm
10:27:12.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.470362 24458 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:12.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.476692 24458 webserver.cc:458] Webserver started at http://127.23.187.124:32989/ using document root <none> and password file <none>
10:27:12.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.477803 24458 fs_manager.cc:362] Metadata directory not provided
10:27:12.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.477990 24458 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:12.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.478374 24458 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:27:12.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.483232 24458 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-2/data/instance:
10:27:12.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5484110be13c402aac7c62daa8c27f87"
10:27:12.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:12 on dist-test-slave-hht6"
10:27:12.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.484328 24458 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-2/wal/instance:
10:27:12.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5484110be13c402aac7c62daa8c27f87"
10:27:12.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:12 on dist-test-slave-hht6"
10:27:12.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.491701 24458 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
10:27:12.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.499567 24483 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:12.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.501456 24458 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.001s
10:27:12.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.501747 24458 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-2/data,/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-2/wal
10:27:12.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5484110be13c402aac7c62daa8c27f87"
10:27:12.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:12 on dist-test-slave-hht6"
10:27:12.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.502058 24458 fs_report.cc:389] FS layout report
10:27:12.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:12.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-2/wal
10:27:12.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-2/wal
10:27:12.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-2/data/data
10:27:12.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:12.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:12.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:12.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:12.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:12.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:12.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:12.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:12.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:12.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:12.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:12.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.551337 24458 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:27:12.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.552786 24458 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:12.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.553225 24458 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:12.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.623768 24458 rpc_server.cc:307] RPC server started. Bound to: 127.23.187.124:38041
10:27:12.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.623899 24534 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.187.124:38041 every 8 connection(s)
10:27:12.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.626555 24458 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-2/data/info.pb
10:27:12.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.631652 24535 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:27:12.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.634428 24301 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu as pid 24458
10:27:12.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.634918 24301 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/master-2/wal/instance
10:27:12.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.648666 24535 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.187.126" port: 41049 } has no permanent_uuid. Determining permanent_uuid...
10:27:12.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.676846 24535 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } has no permanent_uuid. Determining permanent_uuid...
10:27:12.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.686287 24535 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } has no permanent_uuid. Determining permanent_uuid...
10:27:12.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.702106 24535 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87: Bootstrap starting.
10:27:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.705868 24381 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } attempt: 6
10:27:12.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.710467 24535 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87: Neither blocks nor log segments found. Creating new log.
10:27:12.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.712821 24535 log.cc:826] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87: Log is configured to *not* fsync() on all Append() calls
10:27:12.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.718108 24535 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87: No bootstrap required, opened a new log
10:27:12.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.725126 24381 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e: Bootstrap starting.
10:27:12.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.730427 24381 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e: Neither blocks nor log segments found. Creating new log.
10:27:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.732354 24381 log.cc:826] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e: Log is configured to *not* fsync() on all Append() calls
10:27:12.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.737532 24381 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e: No bootstrap required, opened a new log
10:27:12.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.740078 24535 raft_consensus.cc:357] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3390f8eca7534b5ab3700276aa09044e" member_type: VOTER last_known_addr { host: "127.23.187.126" port: 41049 } } peers { permanent_uuid: "d4ecff1756e24efbb08a49f672be43a8" member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } } peers { permanent_uuid: "5484110be13c402aac7c62daa8c27f87" member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } }
10:27:12.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.740684 24535 raft_consensus.cc:383] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:27:12.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.740878 24535 raft_consensus.cc:738] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5484110be13c402aac7c62daa8c27f87, State: Initialized, Role: FOLLOWER
10:27:12.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.741639 24535 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3390f8eca7534b5ab3700276aa09044e" member_type: VOTER last_known_addr { host: "127.23.187.126" port: 41049 } } peers { permanent_uuid: "d4ecff1756e24efbb08a49f672be43a8" member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } } peers { permanent_uuid: "5484110be13c402aac7c62daa8c27f87" member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } }
10:27:12.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.744176 24546 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3390f8eca7534b5ab3700276aa09044e" member_type: VOTER last_known_addr { host: "127.23.187.126" port: 41049 } } peers { permanent_uuid: "d4ecff1756e24efbb08a49f672be43a8" member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } } peers { permanent_uuid: "5484110be13c402aac7c62daa8c27f87" member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } } }
10:27:12.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.745255 24546 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [sys.catalog]: This master's current role is: FOLLOWER
10:27:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.745805 24535 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [sys.catalog]: configured and running, proceeding with master startup.
10:27:12.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.764492 24381 raft_consensus.cc:357] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3390f8eca7534b5ab3700276aa09044e" member_type: VOTER last_known_addr { host: "127.23.187.126" port: 41049 } } peers { permanent_uuid: "d4ecff1756e24efbb08a49f672be43a8" member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } } peers { permanent_uuid: "5484110be13c402aac7c62daa8c27f87" member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } }
10:27:12.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.765193 24381 raft_consensus.cc:383] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:27:12.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.765493 24381 raft_consensus.cc:738] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3390f8eca7534b5ab3700276aa09044e, State: Initialized, Role: FOLLOWER
10:27:12.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.766330 24381 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3390f8eca7534b5ab3700276aa09044e" member_type: VOTER last_known_addr { host: "127.23.187.126" port: 41049 } } peers { permanent_uuid: "d4ecff1756e24efbb08a49f672be43a8" member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } } peers { permanent_uuid: "5484110be13c402aac7c62daa8c27f87" member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } }
10:27:12.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.769395 24558 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3390f8eca7534b5ab3700276aa09044e" member_type: VOTER last_known_addr { host: "127.23.187.126" port: 41049 } } peers { permanent_uuid: "d4ecff1756e24efbb08a49f672be43a8" member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } } peers { permanent_uuid: "5484110be13c402aac7c62daa8c27f87" member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } } }
10:27:12.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.770419 24558 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e [sys.catalog]: This master's current role is: FOLLOWER
10:27:12.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.771446 24381 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e [sys.catalog]: configured and running, proceeding with master startup.
10:27:12.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:12.776703 24557 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:27:12.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:12.777035 24557 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:27:12.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:12.806380 24335 tablet.cc:2367] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e: Can't schedule compaction. Clean time has not been advanced past its initial value.
10:27:12.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.771339 24458 master_runner.cc:186] Error getting master registration for 127.23.187.126:41049: OK, instance_id { permanent_uuid: "3390f8eca7534b5ab3700276aa09044e" instance_seqno: 1736936829097189 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
10:27:12.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.807582 24458 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:27:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:12.808396 24489 tablet.cc:2367] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87: Can't schedule compaction. Clean time has not been advanced past its initial value.
10:27:12.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:12.824277 24570 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:27:12.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:12.824592 24570 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:27:12.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.822456 24313 master_runner.cc:186] Error getting master registration for 127.23.187.125:45989: OK, instance_id { permanent_uuid: "d4ecff1756e24efbb08a49f672be43a8" instance_seqno: 1736936830843664 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
10:27:12.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:12.858139 24313 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:27:13.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.047464 24546 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:27:13.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.048031 24546 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3390f8eca7534b5ab3700276aa09044e" member_type: VOTER last_known_addr { host: "127.23.187.126" port: 41049 } } peers { permanent_uuid: "d4ecff1756e24efbb08a49f672be43a8" member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } } peers { permanent_uuid: "5484110be13c402aac7c62daa8c27f87" member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } }
10:27:13.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.051788 24546 leader_election.cc:290] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3390f8eca7534b5ab3700276aa09044e (127.23.187.126:41049), d4ecff1756e24efbb08a49f672be43a8 (127.23.187.125:45989)
10:27:13.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.051682 24356 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5484110be13c402aac7c62daa8c27f87" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3390f8eca7534b5ab3700276aa09044e" is_pre_election: true
10:27:13.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.052573 24356 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 5484110be13c402aac7c62daa8c27f87 in term 0.
10:27:13.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.054044 24484 leader_election.cc:304] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3390f8eca7534b5ab3700276aa09044e, 5484110be13c402aac7c62daa8c27f87; no voters:
10:27:13.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.055008 24546 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [term 0 FOLLOWER]: Leader pre-election won for term 1
10:27:13.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.055692 24546 raft_consensus.cc:491] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
10:27:13.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.056377 24546 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [term 0 FOLLOWER]: Advancing to term 1
10:27:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.068086 24546 raft_consensus.cc:513] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3390f8eca7534b5ab3700276aa09044e" member_type: VOTER last_known_addr { host: "127.23.187.126" port: 41049 } } peers { permanent_uuid: "d4ecff1756e24efbb08a49f672be43a8" member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } } peers { permanent_uuid: "5484110be13c402aac7c62daa8c27f87" member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } }
10:27:13.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.072353 24356 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5484110be13c402aac7c62daa8c27f87" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3390f8eca7534b5ab3700276aa09044e"
10:27:13.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.072965 24356 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e [term 0 FOLLOWER]: Advancing to term 1
10:27:13.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.073097 24546 leader_election.cc:290] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [CANDIDATE]: Term 1 election: Requested vote from peers 3390f8eca7534b5ab3700276aa09044e (127.23.187.126:41049), d4ecff1756e24efbb08a49f672be43a8 (127.23.187.125:45989)
10:27:13.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.080209 24356 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 5484110be13c402aac7c62daa8c27f87 in term 1.
10:27:13.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.081681 24484 leader_election.cc:304] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3390f8eca7534b5ab3700276aa09044e, 5484110be13c402aac7c62daa8c27f87; no voters:
10:27:13.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.084060 24546 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [term 1 FOLLOWER]: Leader election won for term 1
10:27:13.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.087379 24546 raft_consensus.cc:695] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [term 1 LEADER]: Becoming Leader. State: Replica: 5484110be13c402aac7c62daa8c27f87, State: Running, Role: LEADER
10:27:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.089673 24546 consensus_queue.cc:237] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3390f8eca7534b5ab3700276aa09044e" member_type: VOTER last_known_addr { host: "127.23.187.126" port: 41049 } } peers { permanent_uuid: "d4ecff1756e24efbb08a49f672be43a8" member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } } peers { permanent_uuid: "5484110be13c402aac7c62daa8c27f87" member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } }
10:27:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.106515 24572 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5484110be13c402aac7c62daa8c27f87. Latest consensus state: current_term: 1 leader_uuid: "5484110be13c402aac7c62daa8c27f87" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3390f8eca7534b5ab3700276aa09044e" member_type: VOTER last_known_addr { host: "127.23.187.126" port: 41049 } } peers { permanent_uuid: "d4ecff1756e24efbb08a49f672be43a8" member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } } peers { permanent_uuid: "5484110be13c402aac7c62daa8c27f87" member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } } }
10:27:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.107553 24572 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [sys.catalog]: This master's current role is: LEADER
10:27:13.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.120011 24574 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
10:27:13.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.128170 24574 catalog_manager.cc:1485] Initializing Kudu cluster ID...
10:27:13.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.152493 24356 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e [term 1 FOLLOWER]: Refusing update from remote peer 5484110be13c402aac7c62daa8c27f87: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
10:27:13.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.154469 24572 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3390f8eca7534b5ab3700276aa09044e" member_type: VOTER last_known_addr { host: "127.23.187.126" port: 41049 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:27:13.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.188606 24558 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5484110be13c402aac7c62daa8c27f87. Latest consensus state: current_term: 1 leader_uuid: "5484110be13c402aac7c62daa8c27f87" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3390f8eca7534b5ab3700276aa09044e" member_type: VOTER last_known_addr { host: "127.23.187.126" port: 41049 } } peers { permanent_uuid: "d4ecff1756e24efbb08a49f672be43a8" member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } } peers { permanent_uuid: "5484110be13c402aac7c62daa8c27f87" member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } } }
10:27:13.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.193234 24558 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e [sys.catalog]: This master's current role is: FOLLOWER
10:27:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.199409 24575 mvcc.cc:204] Tried to move back new op lower bound from 7114493268569956352 to 7114493268358496256. Current Snapshot: MvccSnapshot[applied={T|T < 7114493268569956352}]
10:27:13.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.207763 24576 mvcc.cc:204] Tried to move back new op lower bound from 7114493268569956352 to 7114493268358496256. Current Snapshot: MvccSnapshot[applied={T|T < 7114493268569956352}]
10:27:13.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.213661 24558 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "5484110be13c402aac7c62daa8c27f87" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3390f8eca7534b5ab3700276aa09044e" member_type: VOTER last_known_addr { host: "127.23.187.126" port: 41049 } } peers { permanent_uuid: "d4ecff1756e24efbb08a49f672be43a8" member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } } peers { permanent_uuid: "5484110be13c402aac7c62daa8c27f87" member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } } }
10:27:13.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.214614 24558 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e [sys.catalog]: This master's current role is: FOLLOWER
10:27:13.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.217366 24456 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } attempt: 7
10:27:13.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.218889 24572 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "5484110be13c402aac7c62daa8c27f87" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3390f8eca7534b5ab3700276aa09044e" member_type: VOTER last_known_addr { host: "127.23.187.126" port: 41049 } } peers { permanent_uuid: "d4ecff1756e24efbb08a49f672be43a8" member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } } peers { permanent_uuid: "5484110be13c402aac7c62daa8c27f87" member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } } }
10:27:13.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.219718 24572 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [sys.catalog]: This master's current role is: LEADER
10:27:13.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.231809 24574 catalog_manager.cc:1348] Generated new cluster ID: d54ae52810814fc0ba388581a62feb1b
10:27:13.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.232110 24574 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
10:27:13.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.282898 24574 catalog_manager.cc:1371] Generated new certificate authority record
10:27:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.284819 24574 catalog_manager.cc:1505] Loading token signing keys...
10:27:13.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.291944 24456 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d4ecff1756e24efbb08a49f672be43a8: Bootstrap starting.
10:27:13.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.322301 24574 catalog_manager.cc:5899] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87: Generated new TSK 0
10:27:13.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.323454 24574 catalog_manager.cc:1515] Initializing in-progress tserver states...
10:27:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.325632 24456 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d4ecff1756e24efbb08a49f672be43a8: Neither blocks nor log segments found. Creating new log.
10:27:13.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.330785 24456 log.cc:826] T 00000000000000000000000000000000 P d4ecff1756e24efbb08a49f672be43a8: Log is configured to *not* fsync() on all Append() calls
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.345759 24301 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-0/wal
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-0/data
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-0/logs
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-0/data/info.pb
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.187.65:0
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.187.65
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.187.65
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.187.126:41049,127.23.187.125:45989,127.23.187.124:38041
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.187.84:44865
10:27:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:13.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:13.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
10:27:13.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.347350 24456 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d4ecff1756e24efbb08a49f672be43a8: No bootstrap required, opened a new log
10:27:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.387204 24456 raft_consensus.cc:357] T 00000000000000000000000000000000 P d4ecff1756e24efbb08a49f672be43a8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3390f8eca7534b5ab3700276aa09044e" member_type: VOTER last_known_addr { host: "127.23.187.126" port: 41049 } } peers { permanent_uuid: "d4ecff1756e24efbb08a49f672be43a8" member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } } peers { permanent_uuid: "5484110be13c402aac7c62daa8c27f87" member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } }
10:27:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.388114 24456 raft_consensus.cc:383] T 00000000000000000000000000000000 P d4ecff1756e24efbb08a49f672be43a8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:27:13.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.388420 24456 raft_consensus.cc:738] T 00000000000000000000000000000000 P d4ecff1756e24efbb08a49f672be43a8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d4ecff1756e24efbb08a49f672be43a8, State: Initialized, Role: FOLLOWER
10:27:13.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.389513 24456 consensus_queue.cc:260] T 00000000000000000000000000000000 P d4ecff1756e24efbb08a49f672be43a8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3390f8eca7534b5ab3700276aa09044e" member_type: VOTER last_known_addr { host: "127.23.187.126" port: 41049 } } peers { permanent_uuid: "d4ecff1756e24efbb08a49f672be43a8" member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } } peers { permanent_uuid: "5484110be13c402aac7c62daa8c27f87" member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } }
10:27:13.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.398196 24584 sys_catalog.cc:455] T 00000000000000000000000000000000 P d4ecff1756e24efbb08a49f672be43a8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3390f8eca7534b5ab3700276aa09044e" member_type: VOTER last_known_addr { host: "127.23.187.126" port: 41049 } } peers { permanent_uuid: "d4ecff1756e24efbb08a49f672be43a8" member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } } peers { permanent_uuid: "5484110be13c402aac7c62daa8c27f87" member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } } }
10:27:13.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.399029 24584 sys_catalog.cc:458] T 00000000000000000000000000000000 P d4ecff1756e24efbb08a49f672be43a8 [sys.catalog]: This master's current role is: FOLLOWER
10:27:13.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.402070 24456 sys_catalog.cc:564] T 00000000000000000000000000000000 P d4ecff1756e24efbb08a49f672be43a8 [sys.catalog]: configured and running, proceeding with master startup.
10:27:13.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.402700 24429 raft_consensus.cc:3054] T 00000000000000000000000000000000 P d4ecff1756e24efbb08a49f672be43a8 [term 0 FOLLOWER]: Advancing to term 1
10:27:13.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.406389 24430 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5484110be13c402aac7c62daa8c27f87" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d4ecff1756e24efbb08a49f672be43a8"
10:27:13.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.409009 24431 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5484110be13c402aac7c62daa8c27f87" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d4ecff1756e24efbb08a49f672be43a8" is_pre_election: true
10:27:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.415611 24429 raft_consensus.cc:1270] T 00000000000000000000000000000000 P d4ecff1756e24efbb08a49f672be43a8 [term 1 FOLLOWER]: Refusing update from remote peer 5484110be13c402aac7c62daa8c27f87: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
10:27:13.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.417608 24572 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d4ecff1756e24efbb08a49f672be43a8" member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:27:13.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.485025 24584 sys_catalog.cc:455] T 00000000000000000000000000000000 P d4ecff1756e24efbb08a49f672be43a8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5484110be13c402aac7c62daa8c27f87. Latest consensus state: current_term: 1 leader_uuid: "5484110be13c402aac7c62daa8c27f87" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3390f8eca7534b5ab3700276aa09044e" member_type: VOTER last_known_addr { host: "127.23.187.126" port: 41049 } } peers { permanent_uuid: "d4ecff1756e24efbb08a49f672be43a8" member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } } peers { permanent_uuid: "5484110be13c402aac7c62daa8c27f87" member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } } }
10:27:13.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.485824 24584 sys_catalog.cc:458] T 00000000000000000000000000000000 P d4ecff1756e24efbb08a49f672be43a8 [sys.catalog]: This master's current role is: FOLLOWER
10:27:13.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.486939 24595 sys_catalog.cc:455] T 00000000000000000000000000000000 P d4ecff1756e24efbb08a49f672be43a8 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "5484110be13c402aac7c62daa8c27f87" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3390f8eca7534b5ab3700276aa09044e" member_type: VOTER last_known_addr { host: "127.23.187.126" port: 41049 } } peers { permanent_uuid: "d4ecff1756e24efbb08a49f672be43a8" member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } } peers { permanent_uuid: "5484110be13c402aac7c62daa8c27f87" member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } } }
10:27:13.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.487895 24595 sys_catalog.cc:458] T 00000000000000000000000000000000 P d4ecff1756e24efbb08a49f672be43a8 [sys.catalog]: This master's current role is: FOLLOWER
10:27:13.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.500191 24601 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
10:27:13.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.830423 24570 catalog_manager.cc:1260] Loaded cluster ID: d54ae52810814fc0ba388581a62feb1b
10:27:13.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.831032 24570 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e: loading cluster ID for follower catalog manager: success
10:27:13.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.839177 24570 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e: acquiring CA information for follower catalog manager: success
10:27:13.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.852120 24570 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 3390f8eca7534b5ab3700276aa09044e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:27:13.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.861194 24572 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "5484110be13c402aac7c62daa8c27f87" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3390f8eca7534b5ab3700276aa09044e" member_type: VOTER last_known_addr { host: "127.23.187.126" port: 41049 } } peers { permanent_uuid: "d4ecff1756e24efbb08a49f672be43a8" member_type: VOTER last_known_addr { host: "127.23.187.125" port: 45989 } } peers { permanent_uuid: "5484110be13c402aac7c62daa8c27f87" member_type: VOTER last_known_addr { host: "127.23.187.124" port: 38041 } } }
10:27:13.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:13.862284 24572 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5484110be13c402aac7c62daa8c27f87 [sys.catalog]: This master's current role is: LEADER
10:27:14.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:14.004541 24583 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:14.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:14.005199 24583 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:14.005963 24583 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:14.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:14.062157 24583 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:14.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:14.063570 24583 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.187.65
10:27:14.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:14.126060 24583 tablet_server_runner.cc:78] Tablet server non-default flags:
10:27:14.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:14.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.187.84:44865
10:27:14.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:14.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:14.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-0/data
10:27:14.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-0/wal
10:27:14.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:14.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:14.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.187.65:0
10:27:14.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:14.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:14.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:14.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-0/data/info.pb
10:27:14.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.187.65
10:27:14.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.187.126:41049,127.23.187.125:45989,127.23.187.124:38041
10:27:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.187.65
10:27:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-0/logs
10:27:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:27:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:27:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:27:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:27:14.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:14.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:14.127796 24583 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:14.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:14.129922 24583 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:27:14.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:14.172770 24611 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:14.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:14.547606 24601 catalog_manager.cc:1260] Loaded cluster ID: d54ae52810814fc0ba388581a62feb1b
10:27:14.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:14.548843 24601 catalog_manager.cc:1553] T 00000000000000000000000000000000 P d4ecff1756e24efbb08a49f672be43a8: loading cluster ID for follower catalog manager: success
10:27:14.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:14.577237 24601 catalog_manager.cc:1575] T 00000000000000000000000000000000 P d4ecff1756e24efbb08a49f672be43a8: acquiring CA information for follower catalog manager: success
10:27:14.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:14.596450 24601 catalog_manager.cc:1603] T 00000000000000000000000000000000 P d4ecff1756e24efbb08a49f672be43a8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:27:15.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:14.179397 24610 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:15.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:14.186600 24614 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:15.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:15.602741 24612 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1428 milliseconds
10:27:15.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.603128 24583 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:27:15.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.607189 24583 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:15.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.609858 24583 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:15.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.611274 24583 hybrid_clock.cc:648] HybridClock initialized: now 1736936835611239 us; error 41 us; skew 500 ppm
10:27:15.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.612036 24583 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:15.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.619397 24583 webserver.cc:458] Webserver started at http://127.23.187.65:38183/ using document root <none> and password file <none>
10:27:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.620306 24583 fs_manager.cc:362] Metadata directory not provided
10:27:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.620530 24583 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:15.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.621031 24583 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:27:15.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.626061 24583 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-0/data/instance:
10:27:15.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "633ccace98f84a8c894376659455d7db"
10:27:15.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:15 on dist-test-slave-hht6"
10:27:15.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.627354 24583 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-0/wal/instance:
10:27:15.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "633ccace98f84a8c894376659455d7db"
10:27:15.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:15 on dist-test-slave-hht6"
10:27:15.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.635346 24583 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
10:27:15.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.641606 24623 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.642694 24583 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
10:27:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.643014 24583 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-0/data,/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-0/wal
10:27:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "633ccace98f84a8c894376659455d7db"
10:27:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:15 on dist-test-slave-hht6"
10:27:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.643342 24583 fs_report.cc:389] FS layout report
10:27:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-0/wal
10:27:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-0/wal
10:27:15.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-0/data/data
10:27:15.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:15.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:15.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:15.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:15.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:15.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:15.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:15.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:15.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:15.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:15.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:15.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.705348 24583 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:27:15.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.706880 24583 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:15.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.707368 24583 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:15.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.710536 24583 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
10:27:15.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.715422 24583 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
10:27:15.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.715638 24583 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
10:27:15.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.715931 24583 ts_tablet_manager.cc:610] Registered 0 tablets
10:27:15.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.716097 24583 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
10:27:15.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.897193 24583 rpc_server.cc:307] RPC server started. Bound to: 127.23.187.65:41157
10:27:15.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.897347 24735 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.187.65:41157 every 8 connection(s)
10:27:15.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.900187 24583 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-0/data/info.pb
10:27:15.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.910487 24301 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu as pid 24583
10:27:15.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.911099 24301 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-0/wal/instance
10:27:15.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.928959 24301 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:15.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:15.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-1/wal
10:27:15.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-1/data
10:27:15.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:15.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:15.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:15.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:15.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:15.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:15.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-1/logs
10:27:15.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-1/data/info.pb
10:27:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:27:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.187.66:0
10:27:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.187.66
10:27:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.187.66
10:27:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.187.126:41049,127.23.187.125:45989,127.23.187.124:38041
10:27:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.187.84:44865
10:27:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
10:27:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.962615 24737 heartbeater.cc:346] Connected to a master server at 127.23.187.125:45989
10:27:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.962747 24736 heartbeater.cc:346] Connected to a master server at 127.23.187.124:38041
10:27:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.963147 24737 heartbeater.cc:463] Registering TS with master...
10:27:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.963551 24736 heartbeater.cc:463] Registering TS with master...
10:27:15.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.964506 24737 heartbeater.cc:509] Master 127.23.187.125:45989 requested a full tablet report, sending...
10:27:15.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.964589 24736 heartbeater.cc:509] Master 127.23.187.124:38041 requested a full tablet report, sending...
10:27:15.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.964646 24738 heartbeater.cc:346] Connected to a master server at 127.23.187.126:41049
10:27:15.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.965066 24738 heartbeater.cc:463] Registering TS with master...
10:27:15.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.965909 24738 heartbeater.cc:509] Master 127.23.187.126:41049 requested a full tablet report, sending...
10:27:15.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.967494 24500 ts_manager.cc:194] Registered new tserver with Master: 633ccace98f84a8c894376659455d7db (127.23.187.65:41157)
10:27:15.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.969790 24421 ts_manager.cc:194] Registered new tserver with Master: 633ccace98f84a8c894376659455d7db (127.23.187.65:41157)
10:27:15.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.969959 24500 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.187.65:59081
10:27:15.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:15.970291 24346 ts_manager.cc:194] Registered new tserver with Master: 633ccace98f84a8c894376659455d7db (127.23.187.65:41157)
10:27:16.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:16.349736 24746 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:16.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:16.350287 24746 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:16.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:16.350814 24746 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:16.382539 24746 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:16.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:16.383455 24746 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.187.66
10:27:16.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:16.419118 24746 tablet_server_runner.cc:78] Tablet server non-default flags:
10:27:16.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:16.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.187.84:44865
10:27:16.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:16.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:16.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-1/data
10:27:16.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-1/wal
10:27:16.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:16.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:16.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.187.66:0
10:27:16.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:16.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:16.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:16.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-1/data/info.pb
10:27:16.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.187.66
10:27:16.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:16.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.187.126:41049,127.23.187.125:45989,127.23.187.124:38041
10:27:16.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:16.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:16.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:16.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:16.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:16.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.187.66
10:27:16.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-1/logs
10:27:16.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:16.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:16.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:16.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:27:16.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:16.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:27:16.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:16.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:27:16.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:27:16.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:16.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:16.420766 24746 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:16.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:16.422549 24746 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:27:16.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:16.441985 24753 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:16.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:16.973194 24736 heartbeater.cc:501] Master 127.23.187.124:38041 was elected leader, sending a full tablet report...
10:27:17.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:16.442688 24752 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:17.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:16.445147 24746 server_base.cc:1034] running on GCE node
10:27:17.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:16.442729 24755 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:17.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.644929 24746 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:17.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.647631 24746 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:17.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.649049 24746 hybrid_clock.cc:648] HybridClock initialized: now 1736936837649011 us; error 53 us; skew 500 ppm
10:27:17.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.649843 24746 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:17.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.656597 24746 webserver.cc:458] Webserver started at http://127.23.187.66:38701/ using document root <none> and password file <none>
10:27:17.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.657549 24746 fs_manager.cc:362] Metadata directory not provided
10:27:17.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.657788 24746 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:17.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.658236 24746 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:27:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.663106 24746 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-1/data/instance:
10:27:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ffcd90944d7e4ae1ad24f5dbf1f3a223"
10:27:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:17 on dist-test-slave-hht6"
10:27:17.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.664233 24746 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-1/wal/instance:
10:27:17.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ffcd90944d7e4ae1ad24f5dbf1f3a223"
10:27:17.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:17 on dist-test-slave-hht6"
10:27:17.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.670914 24746 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.004s
10:27:17.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.676100 24763 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:17.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.677253 24746 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
10:27:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.677628 24746 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-1/data,/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-1/wal
10:27:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ffcd90944d7e4ae1ad24f5dbf1f3a223"
10:27:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:17 on dist-test-slave-hht6"
10:27:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.678040 24746 fs_report.cc:389] FS layout report
10:27:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-1/wal
10:27:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-1/wal
10:27:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-1/data/data
10:27:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:17.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:17.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.728920 24746 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:27:17.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.730438 24746 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:17.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.730887 24746 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:17.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.733480 24746 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
10:27:17.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.738350 24746 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
10:27:17.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.738575 24746 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
10:27:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.738853 24746 ts_tablet_manager.cc:610] Registered 0 tablets
10:27:17.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.739022 24746 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
10:27:17.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.889434 24746 rpc_server.cc:307] RPC server started. Bound to: 127.23.187.66:36155
10:27:17.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.889546 24876 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.187.66:36155 every 8 connection(s)
10:27:17.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.892122 24746 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-1/data/info.pb
10:27:17.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.900458 24301 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu as pid 24746
10:27:17.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.901237 24301 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-1/wal/instance
10:27:17.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.933145 24301 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:17.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:17.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-2/wal
10:27:17.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-2/data
10:27:17.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:17.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:17.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:17.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:17.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:17.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:17.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-2/logs
10:27:17.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-2/data/info.pb
10:27:17.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:17.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:17.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:17.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:17.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:17.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:17.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:17.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:17.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
10:27:17.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:17.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.187.67:0
10:27:17.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.187.67
10:27:17.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.187.67
10:27:17.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:17.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.187.126:41049,127.23.187.125:45989,127.23.187.124:38041
10:27:17.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.187.84:44865
10:27:17.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:17.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:17.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
10:27:17.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.939080 24877 heartbeater.cc:346] Connected to a master server at 127.23.187.124:38041
10:27:17.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.939695 24877 heartbeater.cc:463] Registering TS with master...
10:27:17.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.941268 24877 heartbeater.cc:509] Master 127.23.187.124:38041 requested a full tablet report, sending...
10:27:17.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.944617 24500 ts_manager.cc:194] Registered new tserver with Master: ffcd90944d7e4ae1ad24f5dbf1f3a223 (127.23.187.66:36155)
10:27:17.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.946375 24500 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.187.66:44783
10:27:17.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.948163 24878 heartbeater.cc:346] Connected to a master server at 127.23.187.125:45989
10:27:17.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.948760 24878 heartbeater.cc:463] Registering TS with master...
10:27:17.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.949577 24878 heartbeater.cc:509] Master 127.23.187.125:45989 requested a full tablet report, sending...
10:27:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.952394 24421 ts_manager.cc:194] Registered new tserver with Master: ffcd90944d7e4ae1ad24f5dbf1f3a223 (127.23.187.66:36155)
10:27:17.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.957267 24879 heartbeater.cc:346] Connected to a master server at 127.23.187.126:41049
10:27:17.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.957554 24879 heartbeater.cc:463] Registering TS with master...
10:27:17.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.958055 24879 heartbeater.cc:509] Master 127.23.187.126:41049 requested a full tablet report, sending...
10:27:17.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:17.960700 24346 ts_manager.cc:194] Registered new tserver with Master: ffcd90944d7e4ae1ad24f5dbf1f3a223 (127.23.187.66:36155)
10:27:18.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:18.261648 24887 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:18.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:18.262096 24887 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:18.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:18.262543 24887 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:18.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:18.293219 24887 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:18.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:18.294400 24887 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.187.67
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:18.329620 24887 tablet_server_runner.cc:78] Tablet server non-default flags:
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.187.84:44865
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-2/data
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-2/wal
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.187.67:0
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-2/data/info.pb
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.187.67
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.187.126:41049,127.23.187.125:45989,127.23.187.124:38041
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.187.67
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-2/logs
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:27:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:18.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:18.331139 24887 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:18.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:18.332661 24887 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:27:18.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:18.348030 24895 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:18.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:18.950358 24877 heartbeater.cc:501] Master 127.23.187.124:38041 was elected leader, sending a full tablet report...
10:27:20.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:19.748420 24893 debug-util.cc:398] Leaking SignalData structure 0x7b0800006bc0 after lost signal to thread 24887
10:27:20.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:20.050624 24893 kernel_stack_watchdog.cc:198] Thread 24887 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:636 for 401ms:
10:27:20.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
10:27:20.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
10:27:20.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
10:27:20.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
10:27:20.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:18.348043 24894 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:20.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:20.051596 24887 thread.cc:635] OpenStack (cloud detector) Time spent creating pthread: real 1.707s user 0.641s sys 1.041s
10:27:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:20.051952 24887 thread.cc:601] OpenStack (cloud detector) Time spent starting thread: real 1.707s user 0.641s sys 1.041s
10:27:20.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:20.053079 24897 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:20.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:20.055202 24896 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1705 milliseconds
10:27:20.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.055228 24887 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:27:20.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.056627 24887 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:20.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.059223 24887 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:20.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.060613 24887 hybrid_clock.cc:648] HybridClock initialized: now 1736936840060564 us; error 45 us; skew 500 ppm
10:27:20.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.061491 24887 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:20.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.067579 24887 webserver.cc:458] Webserver started at http://127.23.187.67:35559/ using document root <none> and password file <none>
10:27:20.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.068478 24887 fs_manager.cc:362] Metadata directory not provided
10:27:20.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.068704 24887 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.069273 24887 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:27:20.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.074055 24887 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-2/data/instance:
10:27:20.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "316fc38fbc4349c18b0f87659fe5fa52"
10:27:20.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:20 on dist-test-slave-hht6"
10:27:20.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.075028 24887 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-2/wal/instance:
10:27:20.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "316fc38fbc4349c18b0f87659fe5fa52"
10:27:20.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:20 on dist-test-slave-hht6"
10:27:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.082211 24887 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.002s
10:27:20.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.087692 24908 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:20.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.088673 24887 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
10:27:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.089010 24887 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-2/data,/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-2/wal
10:27:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "316fc38fbc4349c18b0f87659fe5fa52"
10:27:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:20 on dist-test-slave-hht6"
10:27:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.089315 24887 fs_report.cc:389] FS layout report
10:27:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-2/wal
10:27:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-2/wal
10:27:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-2/data/data
10:27:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:20.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:20.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:20.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.148394 24887 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:27:20.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.149813 24887 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:20.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.150218 24887 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:20.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.152647 24887 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
10:27:20.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.157016 24887 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
10:27:20.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.157210 24887 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
10:27:20.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.157410 24887 ts_tablet_manager.cc:610] Registered 0 tablets
10:27:20.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.157581 24887 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
10:27:20.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.295871 24887 rpc_server.cc:307] RPC server started. Bound to: 127.23.187.67:38787
10:27:20.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.295982 25020 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.187.67:38787 every 8 connection(s)
10:27:20.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.298357 24887 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-2/data/info.pb
10:27:20.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.303530 24301 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu as pid 24887
10:27:20.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.304103 24301 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster8633554439466495847/ts-2/wal/instance
10:27:20.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.345240 25023 heartbeater.cc:346] Connected to a master server at 127.23.187.126:41049
10:27:20.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.345274 25022 heartbeater.cc:346] Connected to a master server at 127.23.187.125:45989
10:27:20.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.345604 25021 heartbeater.cc:346] Connected to a master server at 127.23.187.124:38041
10:27:20.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.345794 25023 heartbeater.cc:463] Registering TS with master...
10:27:20.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.345794 25022 heartbeater.cc:463] Registering TS with master...
10:27:20.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.345950 25021 heartbeater.cc:463] Registering TS with master...
10:27:20.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.347064 25021 heartbeater.cc:509] Master 127.23.187.124:38041 requested a full tablet report, sending...
10:27:20.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.347100 25022 heartbeater.cc:509] Master 127.23.187.125:45989 requested a full tablet report, sending...
10:27:20.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.347196 25023 heartbeater.cc:509] Master 127.23.187.126:41049 requested a full tablet report, sending...
10:27:20.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.349941 24500 ts_manager.cc:194] Registered new tserver with Master: 316fc38fbc4349c18b0f87659fe5fa52 (127.23.187.67:38787)
10:27:20.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.350189 24346 ts_manager.cc:194] Registered new tserver with Master: 316fc38fbc4349c18b0f87659fe5fa52 (127.23.187.67:38787)
10:27:20.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.350431 24421 ts_manager.cc:194] Registered new tserver with Master: 316fc38fbc4349c18b0f87659fe5fa52 (127.23.187.67:38787)
10:27:20.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.351692 24500 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.187.67:60677
10:27:20.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.363776 24301 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
10:27:20.374 [DEBUG - main] (MiniKuduCluster.java:195) Response:
10:27:20.374 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
10:27:20.375 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.23.187.126"
port: 41049
}
bound_http_address {
host: "127.23.187.126"
port: 46171
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.23.187.125"
port: 45989
}
bound_http_address {
host: "127.23.187.125"
port: 40005
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.23.187.124"
port: 38041
}
bound_http_address {
host: "127.23.187.124"
port: 32989
}
}
}
10:27:20.375 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
10:27:20.376 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.23.187.65"
port: 41157
}
bound_http_address {
host: "127.23.187.65"
port: 38183
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.23.187.66"
port: 36155
}
bound_http_address {
host: "127.23.187.66"
port: 38701
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.23.187.67"
port: 38787
}
bound_http_address {
host: "127.23.187.67"
port: 35559
}
}
}
10:27:20.376 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
10:27:20.377 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster1854403562239682293/krb5kdc/krb5cc
10:27:20.378 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
10:27:20.378 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskaxjw05/test-tmp
10:27:20.379 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
10:27:20.379 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskaxjw05/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
10:27:20.381 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
10:27:20.709 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:27:20.709 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
10:27:20.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.712050 25032 mini_oidc.cc:150] Starting JWKS server
10:27:20.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.715544 25032 webserver.cc:458] Webserver started at http://127.0.0.1:44767/ using document root <none> and password file <none>
10:27:20.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.715962 25032 mini_oidc.cc:186] Starting OIDC Discovery server
10:27:20.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:20.717772 25032 webserver.cc:458] Webserver started at http://127.0.0.1:42843/ using document root <none> and password file <none>
10:27:20.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
10:27:20.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/principal' for realm 'KRBTEST.COM',
10:27:20.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
10:27:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:20 dist-test-slave-hht6 krb5kdc[25043](info): setting up network...
10:27:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
10:27:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:20 dist-test-slave-hht6 krb5kdc[25043](info): set up 2 sockets
10:27:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:20 dist-test-slave-hht6 krb5kdc[25043](info): commencing operation
10:27:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
10:27:21.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:21.354378 25021 heartbeater.cc:501] Master 127.23.187.124:38041 was elected leader, sending a full tablet report...
10:27:21.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:21.829150 25032 mini_kdc.cc:121] Time spent starting KDC: real 1.072s user 0.000s sys 0.020s
10:27:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
10:27:21.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
10:27:21.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
10:27:21.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
10:27:21.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
10:27:21.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
10:27:21.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
10:27:21.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
10:27:21.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
10:27:21.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
10:27:21.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/test-user.keytab.
10:27:21.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/test-user.keytab.
10:27:21.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:21 dist-test-slave-hht6 krb5kdc[25043](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936841, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:27:21.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
10:27:21.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-15T10:27:21Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
10:27:21.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-15T10:27:21Z Disabled control of system clock
10:27:21.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.24.114.62@KRBTEST.COM; defaulting to no policy
10:27:21.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
10:27:21.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.24.114.62@KRBTEST.COM" created.
10:27:22.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
10:27:22.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.114.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.62.keytab.
10:27:22.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.114.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.62.keytab.
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:22.018993 25032 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/wal
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/logs
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data/info.pb
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.62:41373
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.62
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.62.keytab
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.62
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
10:27:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
10:27:22.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:22.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:22.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:22.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:22.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:22.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:22.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:22.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:22.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
10:27:22.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:22.328261 25057 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:22.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:22.328900 25057 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:22.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:22.329588 25057 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:22.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:22.363583 25057 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
10:27:22.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:22.363867 25057 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:27:22.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:22.364256 25057 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:22.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:22.364595 25057 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:27:22.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:22.364888 25057 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:27:22.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:22.365162 25057 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:27:22.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:22.365371 25057 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44767/jwks.json
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:22.402735 25057 master_runner.cc:386] Master server non-default flags:
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/wal
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.62.keytab
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.62
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.62:41373
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data/info.pb
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.62
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:22.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:22.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/logs
10:27:22.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:22.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:22.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:22.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:27:22.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:22.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:27:22.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:22.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:27:22.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:27:22.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:22.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:22.404353 25057 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:22.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:22.405370 25057 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:27:22.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:22.419863 25064 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:23.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:22.420537 25063 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:23.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:22.422255 25057 server_base.cc:1034] running on GCE node
10:27:23.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:22.421839 25066 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:23.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.619470 25057 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:23.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.622674 25057 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:23.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.624127 25057 hybrid_clock.cc:648] HybridClock initialized: now 1736936843624072 us; error 67 us; skew 500 ppm
10:27:23.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.624984 25057 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:23.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:23 dist-test-slave-hht6 krb5kdc[25043](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936843, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:27:23.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.635427 25057 init.cc:377] Logged in from keytab as kudu/127.24.114.62@KRBTEST.COM (short username kudu)
10:27:23.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.641616 25057 webserver.cc:458] Webserver started at http://127.24.114.62:39231/ using document root <none> and password file <none>
10:27:23.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.642809 25057 fs_manager.cc:362] Metadata directory not provided
10:27:23.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.643179 25057 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:23.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.643626 25057 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:27:23.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.649087 25057 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data/instance:
10:27:23.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "584493ef8c844a048f121a878c0e4745"
10:27:23.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:23 on dist-test-slave-hht6"
10:27:23.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.650126 25057 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/wal/instance:
10:27:23.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "584493ef8c844a048f121a878c0e4745"
10:27:23.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:23 on dist-test-slave-hht6"
10:27:23.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.657243 25057 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
10:27:23.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.662374 25075 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.663296 25057 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.003s
10:27:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.663589 25057 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data,/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/wal
10:27:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "584493ef8c844a048f121a878c0e4745"
10:27:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:23 on dist-test-slave-hht6"
10:27:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.663869 25057 fs_report.cc:389] FS layout report
10:27:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/wal
10:27:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/wal
10:27:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data/data
10:27:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:23.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.711048 25057 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:27:23.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.716517 25057 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:23.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.717128 25057 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:23.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.787724 25057 rpc_server.cc:307] RPC server started. Bound to: 127.24.114.62:41373
10:27:23.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.787803 25126 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.114.62:41373 every 8 connection(s)
10:27:23.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.790351 25057 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data/info.pb
10:27:23.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.795676 25127 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:27:23.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.799168 25032 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu as pid 25057
10:27:23.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.799633 25032 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/wal/instance
10:27:23.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.807606 25127 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } has no permanent_uuid. Determining permanent_uuid...
10:27:23.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.24.114.61@KRBTEST.COM; defaulting to no policy
10:27:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
10:27:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.24.114.61@KRBTEST.COM" created.
10:27:23.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:23 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936843, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.62@KRBTEST.COM for kudu/127.24.114.62@KRBTEST.COM
10:27:23.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
10:27:23.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.114.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.61.keytab.
10:27:23.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.114.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.61.keytab.
10:27:23.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.843401 25032 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:23.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:23.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/wal
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/logs
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data/info.pb
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.61:42113
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.61
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.61.keytab
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.61
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
10:27:23.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.845340 25127 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } has no permanent_uuid. Determining permanent_uuid...
10:27:23.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:23.847906 25078 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.114.61:42113: connect: Connection refused (error 111)
10:27:23.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:23.852360 25127 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.114.61:42113: Network error: Client connection negotiation failed: client connection to 127.24.114.61:42113: connect: Connection refused (error 111)
10:27:23.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.901912 25127 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } attempt: 1
10:27:23.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:23.906071 25127 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.114.61:42113: Network error: Client connection negotiation failed: client connection to 127.24.114.61:42113: connect: Connection refused (error 111)
10:27:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:23.974501 25127 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } attempt: 2
10:27:23.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:23.979430 25127 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.114.61:42113: Network error: Client connection negotiation failed: client connection to 127.24.114.61:42113: connect: Connection refused (error 111)
10:27:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:24.070763 25127 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } attempt: 3
10:27:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:24.074599 25127 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.114.61:42113: Network error: Client connection negotiation failed: client connection to 127.24.114.61:42113: connect: Connection refused (error 111)
10:27:24.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:24.161046 25133 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:24.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:24.161676 25133 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:24.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:24.162331 25133 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:24.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:24.193715 25133 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
10:27:24.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:24.193970 25133 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:27:24.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:24.194458 25133 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:24.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:24.194820 25133 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:27:24.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:24.195094 25133 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:27:24.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:24.195358 25133 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:27:24.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:24.195561 25133 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44767/jwks.json
10:27:24.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:24.217950 25127 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } attempt: 4
10:27:24.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:24.221912 25127 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.114.61:42113: Network error: Client connection negotiation failed: client connection to 127.24.114.61:42113: connect: Connection refused (error 111)
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:24.233753 25133 master_runner.cc:386] Master server non-default flags:
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/wal
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.61.keytab
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.61
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.61:42113
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data/info.pb
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.61
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/logs
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:27:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:27:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:27:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:27:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:24.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:24.234961 25133 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:24.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:24.235939 25133 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:27:24.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:24.249840 25141 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:24.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:24.521296 25127 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } attempt: 5
10:27:24.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:24.528098 25127 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.114.61:42113: Network error: Client connection negotiation failed: client connection to 127.24.114.61:42113: connect: Connection refused (error 111)
10:27:25.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.075693 25127 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } attempt: 6
10:27:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:25.088770 25127 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.114.61:42113: Network error: Client connection negotiation failed: client connection to 127.24.114.61:42113: connect: Connection refused (error 111)
10:27:25.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:24.251688 25144 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:25.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:24.250349 25142 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:25.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:25.563499 25143 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1310 milliseconds
10:27:25.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.563575 25133 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:27:25.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.564924 25133 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.567823 25133 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:25.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.569176 25133 hybrid_clock.cc:648] HybridClock initialized: now 1736936845569139 us; error 44 us; skew 500 ppm
10:27:25.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.570053 25133 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:25.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:25 dist-test-slave-hht6 krb5kdc[25043](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936845, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:27:25.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.576153 25133 init.cc:377] Logged in from keytab as kudu/127.24.114.61@KRBTEST.COM (short username kudu)
10:27:25.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.581403 25133 webserver.cc:458] Webserver started at http://127.24.114.61:42711/ using document root <none> and password file <none>
10:27:25.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.582288 25133 fs_manager.cc:362] Metadata directory not provided
10:27:25.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.582479 25133 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:25.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.582852 25133 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:27:25.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.587661 25133 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data/instance:
10:27:25.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1242297439664bee887da43b847e2418"
10:27:25.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:25 on dist-test-slave-hht6"
10:27:25.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.588708 25133 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/wal/instance:
10:27:25.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1242297439664bee887da43b847e2418"
10:27:25.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:25 on dist-test-slave-hht6"
10:27:25.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.596419 25133 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
10:27:25.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.602238 25154 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.603202 25133 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.003s
10:27:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.603580 25133 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data,/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/wal
10:27:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1242297439664bee887da43b847e2418"
10:27:25.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:25 on dist-test-slave-hht6"
10:27:25.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.603914 25133 fs_report.cc:389] FS layout report
10:27:25.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:25.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/wal
10:27:25.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/wal
10:27:25.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data/data
10:27:25.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:25.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:25.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:25.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:25.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:25.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:25.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:25.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:25.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:25.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:25.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:25.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.667814 25133 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:27:25.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.673880 25133 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:25.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.674440 25133 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.746562 25133 rpc_server.cc:307] RPC server started. Bound to: 127.24.114.61:42113
10:27:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.746652 25205 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.114.61:42113 every 8 connection(s)
10:27:25.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.749401 25133 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data/info.pb
10:27:25.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.754518 25206 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:27:25.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.754927 25032 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu as pid 25133
10:27:25.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.755347 25032 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/wal/instance
10:27:25.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.24.114.60@KRBTEST.COM; defaulting to no policy
10:27:25.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.769914 25206 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } has no permanent_uuid. Determining permanent_uuid...
10:27:25.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
10:27:25.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.24.114.60@KRBTEST.COM" created.
10:27:25.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:25 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936845, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.61@KRBTEST.COM for kudu/127.24.114.62@KRBTEST.COM
10:27:25.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
10:27:25.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.114.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.60.keytab.
10:27:25.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.114.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.60.keytab.
10:27:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.799459 25032 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:25.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/wal
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/logs
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data/info.pb
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.60:43033
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.60
10:27:25.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.60.keytab
10:27:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.60
10:27:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
10:27:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
10:27:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
10:27:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.805743 25206 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } has no permanent_uuid. Determining permanent_uuid...
10:27:25.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:25 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936845, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.61@KRBTEST.COM for kudu/127.24.114.61@KRBTEST.COM
10:27:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.833113 25206 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } has no permanent_uuid. Determining permanent_uuid...
10:27:25.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:25.835704 25155 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:25.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:25.839963 25206 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.114.60:43033: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:25.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.889444 25206 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } attempt: 1
10:27:25.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:25.894641 25206 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.114.60:43033: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:25.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:25.963076 25206 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } attempt: 2
10:27:25.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:25.967106 25206 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.114.60:43033: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:26.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:26.058460 25206 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } attempt: 3
10:27:26.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:26.062096 25206 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.114.60:43033: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:26.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:26.130725 25212 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:26.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:26.131260 25212 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:26.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:26.131862 25212 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:26.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:26.149411 25127 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } attempt: 7
10:27:26.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:26 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936843, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.62@KRBTEST.COM for kudu/127.24.114.61@KRBTEST.COM
10:27:26.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:26.171943 25212 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
10:27:26.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:26.172262 25212 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:27:26.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:26.172262 25127 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } has no permanent_uuid. Determining permanent_uuid...
10:27:26.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:26.172744 25212 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:26.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:26.173092 25212 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:27:26.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:26.173450 25212 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:27:26.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:26.173794 25212 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:27:26.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:26.174067 25212 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44767/jwks.json
10:27:26.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:26.177318 25127 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.114.60:43033: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:26.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:26.205446 25206 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } attempt: 4
10:27:26.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:26.209460 25206 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.114.60:43033: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:26.214908 25212 master_runner.cc:386] Master server non-default flags:
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/wal
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.60.keytab
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.60
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.60:43033
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data/info.pb
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.60
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/logs
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:27:26.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:26.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:26.216105 25212 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:26.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:26.217247 25212 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:27:26.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:26.231671 25223 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:26.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:26.235783 25127 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } attempt: 1
10:27:26.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:26.241436 25127 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.114.60:43033: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:26.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:26.323010 25127 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } attempt: 2
10:27:26.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:26.330482 25127 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.114.60:43033: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:26.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:26.416057 25127 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } attempt: 3
10:27:26.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:26.422338 25127 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.114.60:43033: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:26.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:26.508898 25206 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } attempt: 5
10:27:26.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:26.517078 25206 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.114.60:43033: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:26.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:26.562963 25127 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } attempt: 4
10:27:26.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:26.569294 25127 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.114.60:43033: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:26.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:26.852864 25127 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } attempt: 5
10:27:26.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:26.860126 25127 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.114.60:43033: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.064615 25206 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } attempt: 6
10:27:27.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:27.074644 25206 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.114.60:43033: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:27.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.412770 25127 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } attempt: 6
10:27:27.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:27.421129 25127 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.114.60:43033: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:27.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:26.233698 25222 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:27.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:26.233067 25225 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:27.527388 25224 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1294 milliseconds
10:27:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.527506 25212 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:27:27.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.528789 25212 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:27.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.531298 25212 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.532667 25212 hybrid_clock.cc:648] HybridClock initialized: now 1736936847532618 us; error 48 us; skew 500 ppm
10:27:27.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.533563 25212 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:27.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:27 dist-test-slave-hht6 krb5kdc[25043](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936847, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:27:27.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.539641 25212 init.cc:377] Logged in from keytab as kudu/127.24.114.60@KRBTEST.COM (short username kudu)
10:27:27.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.544886 25212 webserver.cc:458] Webserver started at http://127.24.114.60:45607/ using document root <none> and password file <none>
10:27:27.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.545867 25212 fs_manager.cc:362] Metadata directory not provided
10:27:27.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.546082 25212 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:27.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.546533 25212 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
10:27:27.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.551410 25212 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data/instance:
10:27:27.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "563cf286b3ac4c269c5c6359def4bf91"
10:27:27.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:27 on dist-test-slave-hht6"
10:27:27.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.552572 25212 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/wal/instance:
10:27:27.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "563cf286b3ac4c269c5c6359def4bf91"
10:27:27.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:27 on dist-test-slave-hht6"
10:27:27.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.559872 25212 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
10:27:27.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.565295 25239 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.566242 25212 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
10:27:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.566597 25212 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data,/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/wal
10:27:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "563cf286b3ac4c269c5c6359def4bf91"
10:27:27.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:27 on dist-test-slave-hht6"
10:27:27.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.566933 25212 fs_report.cc:389] FS layout report
10:27:27.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:27.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/wal
10:27:27.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/wal
10:27:27.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data/data
10:27:27.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:27.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:27.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:27.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:27.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:27.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:27.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:27.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:27.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:27.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:27.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:27.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.616477 25212 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:27:27.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.621795 25212 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:27.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.622251 25212 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:27.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.703485 25212 rpc_server.cc:307] RPC server started. Bound to: 127.24.114.60:43033
10:27:27.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.703629 25290 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.114.60:43033 every 8 connection(s)
10:27:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.706926 25212 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data/info.pb
10:27:27.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.712862 25292 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
10:27:27.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.714958 25032 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu as pid 25212
10:27:27.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.715310 25032 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/wal/instance
10:27:27.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.727000 25292 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } has no permanent_uuid. Determining permanent_uuid...
10:27:27.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:27 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936847, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.60@KRBTEST.COM for kudu/127.24.114.62@KRBTEST.COM
10:27:27.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:27 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936841, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.114.60@KRBTEST.COM
10:27:27.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.758181 25292 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } has no permanent_uuid. Determining permanent_uuid...
10:27:27.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:27 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936847, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.60@KRBTEST.COM for kudu/127.24.114.61@KRBTEST.COM
10:27:27.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:27 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936841, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.114.61@KRBTEST.COM
10:27:27.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.779672 25292 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } has no permanent_uuid. Determining permanent_uuid...
10:27:27.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:27 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936847, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.60@KRBTEST.COM for kudu/127.24.114.60@KRBTEST.COM
10:27:27.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:27 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936841, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.114.62@KRBTEST.COM
10:27:27.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.811223 25292 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: Bootstrap starting.
10:27:27.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.817831 25292 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: Neither blocks nor log segments found. Creating new log.
10:27:27.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.818511 25032 external_mini_cluster.cc:934] 0 TS(s) registered with all masters
10:27:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.819625 25292 log.cc:826] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: Log is configured to *not* fsync() on all Append() calls
10:27:27.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.824119 25292 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: No bootstrap required, opened a new log
10:27:27.826 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:27:27.826 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
10:27:27.827 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
10:27:27.827 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
10:27:27.828 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.24.114.62"
port: 41373
}
bound_http_address {
host: "127.24.114.62"
port: 39231
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.24.114.61"
port: 42113
}
bound_http_address {
host: "127.24.114.61"
port: 42711
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.24.114.60"
port: 43033
}
bound_http_address {
host: "127.24.114.60"
port: 45607
}
}
}
10:27:27.829 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
10:27:27.830 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:27:27.830 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
10:27:27.830 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
10:27:27.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.850838 25292 raft_consensus.cc:357] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:27.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.851513 25292 raft_consensus.cc:383] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:27:27.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.851768 25292 raft_consensus.cc:738] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 563cf286b3ac4c269c5c6359def4bf91, State: Initialized, Role: FOLLOWER
10:27:27.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.852474 25292 consensus_queue.cc:260] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:27.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.856433 25303 sys_catalog.cc:455] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:27.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.857426 25303 sys_catalog.cc:458] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: This master's current role is: FOLLOWER
10:27:27.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.858155 25292 sys_catalog.cc:564] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: configured and running, proceeding with master startup.
10:27:27.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:27 dist-test-slave-hht6 krb5kdc[25043](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936847, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:27:27.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
10:27:27.866 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:27:27.867 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5cc
10:27:27.868 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
10:27:27.868 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-user
10:27:27.869 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-user"
}
10:27:27.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:27.878196 25245 tablet.cc:2367] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: Can't schedule compaction. Clean time has not been advanced past its initial value.
10:27:27.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:27 dist-test-slave-hht6 krb5kdc[25043](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936847, etypes {rep=17 tkt=17 ses=17}, test-user@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:27:27.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-user@KRBTEST.COM:
10:27:27.909 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:27:27.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:27.915753 25315 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
10:27:27.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:27.916078 25315 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
10:27:27.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.888700 25212 master_runner.cc:186] Error getting master registration for 127.24.114.62:41373: OK, instance_id { permanent_uuid: "584493ef8c844a048f121a878c0e4745" instance_seqno: 1736936843767186 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
10:27:27.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:27.929848 25212 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:27:28.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.135159 25206 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } attempt: 7
10:27:28.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:28 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936845, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.61@KRBTEST.COM for kudu/127.24.114.60@KRBTEST.COM
10:27:28.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.168337 25206 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: Bootstrap starting.
10:27:28.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.176131 25206 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: Neither blocks nor log segments found. Creating new log.
10:27:28.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.178529 25206 log.cc:826] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: Log is configured to *not* fsync() on all Append() calls
10:27:28.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.183488 25206 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: No bootstrap required, opened a new log
10:27:28.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.207450 25303 raft_consensus.cc:491] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:27:28.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.207993 25303 raft_consensus.cc:513] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:28.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.210364 25303 leader_election.cc:290] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 584493ef8c844a048f121a878c0e4745 (127.24.114.62:41373), 1242297439664bee887da43b847e2418 (127.24.114.61:42113)
10:27:28.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.214576 25206 raft_consensus.cc:357] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:28.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.215391 25206 raft_consensus.cc:383] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:27:28.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.215673 25206 raft_consensus.cc:738] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1242297439664bee887da43b847e2418, State: Initialized, Role: FOLLOWER
10:27:28.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.216578 25206 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:28.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.220018 25320 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:28.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.221045 25320 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: This master's current role is: FOLLOWER
10:27:28.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.221923 25206 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: configured and running, proceeding with master startup.
10:27:28.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.222162 25181 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "563cf286b3ac4c269c5c6359def4bf91" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1242297439664bee887da43b847e2418" is_pre_election: true
10:27:28.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.223027 25181 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 563cf286b3ac4c269c5c6359def4bf91 in term 0.
10:27:28.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.224344 25242 leader_election.cc:304] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1242297439664bee887da43b847e2418, 563cf286b3ac4c269c5c6359def4bf91; no voters:
10:27:28.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.225437 25303 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [term 0 FOLLOWER]: Leader pre-election won for term 1
10:27:28.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.225831 25303 raft_consensus.cc:491] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
10:27:28.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.226181 25303 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [term 0 FOLLOWER]: Advancing to term 1
10:27:28.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.233676 25303 raft_consensus.cc:513] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:28.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.237135 25303 leader_election.cc:290] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [CANDIDATE]: Term 1 election: Requested vote from peers 584493ef8c844a048f121a878c0e4745 (127.24.114.62:41373), 1242297439664bee887da43b847e2418 (127.24.114.61:42113)
10:27:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.239385 25181 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "563cf286b3ac4c269c5c6359def4bf91" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1242297439664bee887da43b847e2418"
10:27:28.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.240013 25181 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 0 FOLLOWER]: Advancing to term 1
10:27:28.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.248683 25181 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 563cf286b3ac4c269c5c6359def4bf91 in term 1.
10:27:28.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.250706 25242 leader_election.cc:304] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1242297439664bee887da43b847e2418, 563cf286b3ac4c269c5c6359def4bf91; no voters:
10:27:28.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.251515 25303 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [term 1 FOLLOWER]: Leader election won for term 1
10:27:28.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.253589 25303 raft_consensus.cc:695] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [term 1 LEADER]: Becoming Leader. State: Replica: 563cf286b3ac4c269c5c6359def4bf91, State: Running, Role: LEADER
10:27:28.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.254527 25303 consensus_queue.cc:237] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:28.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.269692 25329 sys_catalog.cc:455] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 563cf286b3ac4c269c5c6359def4bf91. Latest consensus state: current_term: 1 leader_uuid: "563cf286b3ac4c269c5c6359def4bf91" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:28.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.270783 25329 sys_catalog.cc:458] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: This master's current role is: LEADER
10:27:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.276046 25335 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
10:27:28.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.277930 25334 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
10:27:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.285919 25334 catalog_manager.cc:1485] Initializing Kudu cluster ID...
10:27:28.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.280220 25133 master_runner.cc:186] Error getting master registration for 127.24.114.62:41373: OK, instance_id { permanent_uuid: "584493ef8c844a048f121a878c0e4745" instance_seqno: 1736936843767186 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
10:27:28.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.317520 25133 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:27:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.330404 25181 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 1 FOLLOWER]: Refusing update from remote peer 563cf286b3ac4c269c5c6359def4bf91: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
10:27:28.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.332103 25303 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:27:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.356318 25336 mvcc.cc:204] Tried to move back new op lower bound from 7114493330743685120 to 7114493330468732928. Current Snapshot: MvccSnapshot[applied={T|T < 7114493330743685120}]
10:27:28.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.358037 25320 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 563cf286b3ac4c269c5c6359def4bf91. Latest consensus state: current_term: 1 leader_uuid: "563cf286b3ac4c269c5c6359def4bf91" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:28.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.358772 25320 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: This master's current role is: FOLLOWER
10:27:28.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.359974 25303 sys_catalog.cc:455] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "563cf286b3ac4c269c5c6359def4bf91" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:28.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.360738 25303 sys_catalog.cc:458] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: This master's current role is: LEADER
10:27:28.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.362937 25334 catalog_manager.cc:1348] Generated new cluster ID: 3f87294c829d403599dd52ce54e7039e
10:27:28.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.363240 25334 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
10:27:28.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.365728 25320 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "563cf286b3ac4c269c5c6359def4bf91" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:28.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.366497 25320 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: This master's current role is: FOLLOWER
10:27:28.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.394732 25334 catalog_manager.cc:1371] Generated new certificate authority record
10:27:28.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.396848 25334 catalog_manager.cc:1505] Loading token signing keys...
10:27:28.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.418238 25334 catalog_manager.cc:5899] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: Generated new TSK 0
10:27:28.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.419322 25334 catalog_manager.cc:1515] Initializing in-progress tserver states...
10:27:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.454630 25127 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } attempt: 7
10:27:28.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:28 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936843, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.62@KRBTEST.COM for kudu/127.24.114.60@KRBTEST.COM
10:27:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.487103 25127 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: Bootstrap starting.
10:27:28.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.494419 25127 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: Neither blocks nor log segments found. Creating new log.
10:27:28.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.497027 25127 log.cc:826] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: Log is configured to *not* fsync() on all Append() calls
10:27:28.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.503011 25127 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: No bootstrap required, opened a new log
10:27:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.524611 25127 raft_consensus.cc:357] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.525547 25127 raft_consensus.cc:383] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
10:27:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.525851 25127 raft_consensus.cc:738] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 584493ef8c844a048f121a878c0e4745, State: Initialized, Role: FOLLOWER
10:27:28.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.526876 25127 consensus_queue.cc:260] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:28.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.529464 25343 sys_catalog.cc:455] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:28.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.530314 25343 sys_catalog.cc:458] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: This master's current role is: FOLLOWER
10:27:28.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.531235 25127 sys_catalog.cc:564] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: configured and running, proceeding with master startup.
10:27:28.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.531459 25102 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "563cf286b3ac4c269c5c6359def4bf91" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "584493ef8c844a048f121a878c0e4745" is_pre_election: true
10:27:28.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.532330 25102 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 563cf286b3ac4c269c5c6359def4bf91 in term 0.
10:27:28.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.534824 25100 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 0 FOLLOWER]: Advancing to term 1
10:27:28.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.535921 25101 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "563cf286b3ac4c269c5c6359def4bf91" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "584493ef8c844a048f121a878c0e4745"
10:27:28.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.546561 25100 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 1 FOLLOWER]: Refusing update from remote peer 563cf286b3ac4c269c5c6359def4bf91: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
10:27:28.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.548158 25303 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [LEADER]: Connected to new peer: Peer: permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
10:27:28.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.592921 25343 sys_catalog.cc:455] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 563cf286b3ac4c269c5c6359def4bf91. Latest consensus state: current_term: 1 leader_uuid: "563cf286b3ac4c269c5c6359def4bf91" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:28.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.593989 25343 sys_catalog.cc:458] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: This master's current role is: FOLLOWER
10:27:28.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.593602 25355 sys_catalog.cc:455] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "563cf286b3ac4c269c5c6359def4bf91" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:28.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.594476 25355 sys_catalog.cc:458] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: This master's current role is: FOLLOWER
10:27:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:28.595387 25359 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
10:27:29.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:29.155483 25303 sys_catalog.cc:455] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "563cf286b3ac4c269c5c6359def4bf91" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:29.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:29.157127 25303 sys_catalog.cc:458] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: This master's current role is: LEADER
10:27:29.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:29.284212 25335 catalog_manager.cc:1260] Loaded cluster ID: 3f87294c829d403599dd52ce54e7039e
10:27:29.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:29.284510 25335 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: loading cluster ID for follower catalog manager: success
10:27:29.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:29.291846 25335 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: acquiring CA information for follower catalog manager: success
10:27:29.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:29.297591 25335 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:27:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:29.604463 25359 catalog_manager.cc:1260] Loaded cluster ID: 3f87294c829d403599dd52ce54e7039e
10:27:29.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:29.604982 25359 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: loading cluster ID for follower catalog manager: success
10:27:29.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:29.612511 25359 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: acquiring CA information for follower catalog manager: success
10:27:29.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:29.619999 25359 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:27:30.911 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.114.61:42113
10:27:30.912 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
10:27:30.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:30.912928 25032 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu with pid 25133
10:27:30.942 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:27:30.943 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.114.62:41373
10:27:30.943 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
10:27:30.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:30.943430 25032 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu with pid 25057
10:27:30.973 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:27:30.973 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.114.60:43033
10:27:30.973 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
10:27:30.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:30.973789 25032 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu with pid 25212
10:27:30.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:30.996260 24377 debug-util.cc:398] Leaking SignalData structure 0x7b08000452e0 after lost signal to thread 24314
10:27:30.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:30.997080 24377 debug-util.cc:398] Leaking SignalData structure 0x7b080009f2e0 after lost signal to thread 24380
10:27:31.001 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:27:31.001 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.114.61:42113
10:27:31.002 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
10:27:31.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:31.003379 25032 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:31.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:31.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/wal
10:27:31.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data
10:27:31.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:31.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/logs
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data/info.pb
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.61:42113
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.61
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42711
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.61.keytab
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.61
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:31.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:31.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:31.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
10:27:31.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:31.333034 25367 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:31.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:31.333568 25367 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:31.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:31.334157 25367 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:31.366751 25367 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
10:27:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:31.367017 25367 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:27:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:31.367413 25367 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:31.367697 25367 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:27:31.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:31.367964 25367 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:27:31.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:31.368238 25367 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:27:31.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:31.368444 25367 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44767/jwks.json
10:27:31.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:31.409191 25367 master_runner.cc:386] Master server non-default flags:
10:27:31.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:31.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:31.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:31.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:31.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data
10:27:31.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/wal
10:27:31.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
10:27:31.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:31.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:31.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.61.keytab
10:27:31.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.61
10:27:31.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:31.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.61:42113
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data/info.pb
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.61
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42711
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/logs
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:27:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:31.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:31.410844 25367 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:31.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:31.411845 25367 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:27:31.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:31.429723 25374 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:32.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:31.430303 25375 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:32.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:31.433033 25377 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:32.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:32.704361 25376 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
10:27:32.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.704701 25367 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:27:32.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.709229 25367 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:32.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.712833 25367 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:32.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.714321 25367 hybrid_clock.cc:648] HybridClock initialized: now 1736936852714278 us; error 59 us; skew 500 ppm
10:27:32.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.715220 25367 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:32.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:32 dist-test-slave-hht6 krb5kdc[25043](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936852, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:27:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.727782 25367 init.cc:377] Logged in from keytab as kudu/127.24.114.61@KRBTEST.COM (short username kudu)
10:27:32.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.733322 25367 webserver.cc:458] Webserver started at http://127.24.114.61:42711/ using document root <none> and password file <none>
10:27:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.734283 25367 fs_manager.cc:362] Metadata directory not provided
10:27:32.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.734489 25367 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:32.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.743007 25367 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
10:27:32.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.748034 25385 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:32.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.749317 25367 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
10:27:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.749650 25367 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data,/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/wal
10:27:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1242297439664bee887da43b847e2418"
10:27:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:25 on dist-test-slave-hht6"
10:27:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.751952 25367 fs_report.cc:389] FS layout report
10:27:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/wal
10:27:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/wal
10:27:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data/data
10:27:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:32.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.801184 25367 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:27:32.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.807052 25367 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:32.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.807585 25367 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:32.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.885937 25367 rpc_server.cc:307] RPC server started. Bound to: 127.24.114.61:42113
10:27:32.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.886121 25436 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.114.61:42113 every 8 connection(s)
10:27:32.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.889858 25367 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data/info.pb
10:27:32.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.892132 25032 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu as pid 25367
10:27:32.892 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:27:32.893 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.114.62:41373
10:27:32.893 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
10:27:32.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.895274 25032 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:32.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:32.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/wal
10:27:32.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data
10:27:32.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:32.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:32.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/logs
10:27:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data/info.pb
10:27:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:27:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:32.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.62:41373
10:27:32.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.62
10:27:32.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39231
10:27:32.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.62.keytab
10:27:32.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.62
10:27:32.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:32.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
10:27:32.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
10:27:32.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:32.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:32.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:32.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:32.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:32.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:32.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:32.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:32.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
10:27:32.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.906406 25437 sys_catalog.cc:263] Verifying existing consensus state
10:27:32.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.914337 25437 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: Bootstrap starting.
10:27:32.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.953100 25437 log.cc:826] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: Log is configured to *not* fsync() on all Append() calls
10:27:32.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.968376 25437 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:27:32.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.969151 25437 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: Bootstrap complete.
10:27:32.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.995168 25437 raft_consensus.cc:357] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:32.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.997480 25437 raft_consensus.cc:738] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1242297439664bee887da43b847e2418, State: Initialized, Role: FOLLOWER
10:27:32.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:32.998294 25437 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:33.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:33.001158 25441 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:33.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:33.002060 25441 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: This master's current role is: FOLLOWER
10:27:33.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:33.002974 25437 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: configured and running, proceeding with master startup.
10:27:33.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:33.036230 25452 catalog_manager.cc:1260] Loaded cluster ID: 3f87294c829d403599dd52ce54e7039e
10:27:33.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:33.036624 25452 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: loading cluster ID for follower catalog manager: success
10:27:33.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:33.045794 25452 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: acquiring CA information for follower catalog manager: success
10:27:33.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:33.052027 25452 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:27:33.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:33.045293 25367 master_runner.cc:186] Error getting master registration for 127.24.114.62:41373: Network error: Client connection negotiation failed: client connection to 127.24.114.62:41373: connect: Connection refused (error 111),
10:27:33.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:33.075453 25367 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:27:33.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:33.254611 25439 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:33.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:33.255271 25439 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:33.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:33.256072 25439 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:33.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:33.287873 25439 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
10:27:33.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:33.288177 25439 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:27:33.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:33.288517 25439 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:33.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:33.288741 25439 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:27:33.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:33.289023 25439 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:27:33.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:33.289227 25439 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:27:33.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:33.289374 25439 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44767/jwks.json
10:27:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:33.327977 25439 master_runner.cc:386] Master server non-default flags:
10:27:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data
10:27:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/wal
10:27:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
10:27:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.62.keytab
10:27:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.62
10:27:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.62:41373
10:27:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data/info.pb
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.62
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39231
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/logs
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:33.329316 25439 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:33.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:33.330519 25439 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:27:33.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:33.348465 25461 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:33.349366 25463 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:34.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:33.349593 25460 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:34.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:33.350152 25439 server_base.cc:1034] running on GCE node
10:27:34.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.583680 25439 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:34.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.586807 25439 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:34.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.588212 25439 hybrid_clock.cc:648] HybridClock initialized: now 1736936854588139 us; error 76 us; skew 500 ppm
10:27:34.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.589197 25439 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:34.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:34 dist-test-slave-hht6 krb5kdc[25043](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936854, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:27:34.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.595065 25439 init.cc:377] Logged in from keytab as kudu/127.24.114.62@KRBTEST.COM (short username kudu)
10:27:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.600589 25439 webserver.cc:458] Webserver started at http://127.24.114.62:39231/ using document root <none> and password file <none>
10:27:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.601517 25439 fs_manager.cc:362] Metadata directory not provided
10:27:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.601747 25439 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:34.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.609670 25439 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.008s sys 0.000s
10:27:34.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.614780 25472 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:34.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.615897 25439 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
10:27:34.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.616241 25439 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data,/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/wal
10:27:34.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "584493ef8c844a048f121a878c0e4745"
10:27:34.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:23 on dist-test-slave-hht6"
10:27:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.618769 25439 fs_report.cc:389] FS layout report
10:27:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/wal
10:27:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/wal
10:27:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data/data
10:27:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:34.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.693506 25439 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:27:34.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.699331 25439 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:34.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.699805 25439 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:34.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.789194 25439 rpc_server.cc:307] RPC server started. Bound to: 127.24.114.62:41373
10:27:34.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.789409 25523 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.114.62:41373 every 8 connection(s)
10:27:34.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.791996 25439 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data/info.pb
10:27:34.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.796339 25032 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu as pid 25439
10:27:34.796 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:27:34.797 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.114.60:43033
10:27:34.797 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
10:27:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.798791 25032 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/wal
10:27:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data
10:27:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/logs
10:27:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data/info.pb
10:27:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.60:43033
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.60
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45607
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.60.keytab
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.60
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
10:27:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.807747 25524 sys_catalog.cc:263] Verifying existing consensus state
10:27:34.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.813244 25524 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: Bootstrap starting.
10:27:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.834157 25527 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:27:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.835131 25527 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:34.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.855077 25527 leader_election.cc:290] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 584493ef8c844a048f121a878c0e4745 (127.24.114.62:41373), 563cf286b3ac4c269c5c6359def4bf91 (127.24.114.60:43033)
10:27:34.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:34.866765 25386 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:34.874675 25386 leader_election.cc:336] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 563cf286b3ac4c269c5c6359def4bf91 (127.24.114.60:43033): Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:34.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.890317 25524 log.cc:826] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: Log is configured to *not* fsync() on all Append() calls
10:27:34.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:34 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936852, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.61@KRBTEST.COM for kudu/127.24.114.62@KRBTEST.COM
10:27:34.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.985924 25524 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:27:34.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:34.987010 25524 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: Bootstrap complete.
10:27:35.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.020227 25524 raft_consensus.cc:357] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:35.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.023972 25524 raft_consensus.cc:738] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 584493ef8c844a048f121a878c0e4745, State: Initialized, Role: FOLLOWER
10:27:35.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.026623 25524 consensus_queue.cc:260] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:35.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.032441 25533 sys_catalog.cc:455] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:35.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.033835 25533 sys_catalog.cc:458] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: This master's current role is: FOLLOWER
10:27:35.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.034374 25524 sys_catalog.cc:564] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: configured and running, proceeding with master startup.
10:27:35.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.034538 25499 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1242297439664bee887da43b847e2418" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "584493ef8c844a048f121a878c0e4745" is_pre_election: true
10:27:35.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.035495 25499 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1242297439664bee887da43b847e2418 in term 1.
10:27:35.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.037714 25387 leader_election.cc:304] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1242297439664bee887da43b847e2418, 584493ef8c844a048f121a878c0e4745; no voters: 563cf286b3ac4c269c5c6359def4bf91
10:27:35.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.038581 25527 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 1 FOLLOWER]: Leader pre-election won for term 2
10:27:35.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.038930 25527 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
10:27:35.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.039203 25527 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 1 FOLLOWER]: Advancing to term 2
10:27:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.047446 25527 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:35.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.051213 25499 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1242297439664bee887da43b847e2418" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "584493ef8c844a048f121a878c0e4745"
10:27:35.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.051985 25499 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 1 FOLLOWER]: Advancing to term 2
10:27:35.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:35.054584 25386 leader_election.cc:336] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 563cf286b3ac4c269c5c6359def4bf91 (127.24.114.60:43033): Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:35.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.057073 25527 leader_election.cc:290] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [CANDIDATE]: Term 2 election: Requested vote from peers 584493ef8c844a048f121a878c0e4745 (127.24.114.62:41373), 563cf286b3ac4c269c5c6359def4bf91 (127.24.114.60:43033)
10:27:35.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.063652 25499 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1242297439664bee887da43b847e2418 in term 2.
10:27:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.064754 25387 leader_election.cc:304] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1242297439664bee887da43b847e2418, 584493ef8c844a048f121a878c0e4745; no voters: 563cf286b3ac4c269c5c6359def4bf91
10:27:35.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.066182 25527 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 2 FOLLOWER]: Leader election won for term 2
10:27:35.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.076968 25527 raft_consensus.cc:695] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 2 LEADER]: Becoming Leader. State: Replica: 1242297439664bee887da43b847e2418, State: Running, Role: LEADER
10:27:35.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.079424 25527 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:35.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.093389 25536 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1242297439664bee887da43b847e2418. Latest consensus state: current_term: 2 leader_uuid: "1242297439664bee887da43b847e2418" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:35.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.094041 25536 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: This master's current role is: LEADER
10:27:35.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.096722 25541 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
10:27:35.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.105687 25541 catalog_manager.cc:1485] Initializing Kudu cluster ID...
10:27:35.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.108342 25541 catalog_manager.cc:1260] Loaded cluster ID: 3f87294c829d403599dd52ce54e7039e
10:27:35.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.108605 25541 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
10:27:35.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.113950 25541 catalog_manager.cc:1505] Loading token signing keys...
10:27:35.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.118646 25541 catalog_manager.cc:5910] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: Loaded TSK: 0
10:27:35.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.120173 25541 catalog_manager.cc:1515] Initializing in-progress tserver states...
10:27:35.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.133121 25547 catalog_manager.cc:1260] Loaded cluster ID: 3f87294c829d403599dd52ce54e7039e
10:27:35.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.133448 25547 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: loading cluster ID for follower catalog manager: success
10:27:35.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.143013 25547 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: acquiring CA information for follower catalog manager: success
10:27:35.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:35 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936854, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.62@KRBTEST.COM for kudu/127.24.114.62@KRBTEST.COM
10:27:35.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.150393 25547 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:27:35.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.189503 25439 master_runner.cc:186] Error getting master registration for 127.24.114.60:43033: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111),
10:27:35.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.220503 25439 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:27:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:35.263904 25526 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:35.264554 25526 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:35.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:35.265256 25526 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:35.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:35.298484 25526 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
10:27:35.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:35.298749 25526 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:27:35.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:35.299145 25526 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:35.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:35.299471 25526 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:27:35.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:35.299777 25526 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:27:35.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:35.300065 25526 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:27:35.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:35.300284 25526 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44767/jwks.json
10:27:35.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.339699 25526 master_runner.cc:386] Master server non-default flags:
10:27:35.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:35.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:35.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:35.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:35.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data
10:27:35.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/wal
10:27:35.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
10:27:35.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:35.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:35.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.60.keytab
10:27:35.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.60
10:27:35.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:35.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:35.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:35.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.60:43033
10:27:35.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:35.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data/info.pb
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.60
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45607
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/logs
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.341172 25526 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:35.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.342092 25526 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:27:35.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:35.360414 25557 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:35.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.535283 25499 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 2 FOLLOWER]: Refusing update from remote peer 1242297439664bee887da43b847e2418: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
10:27:35.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.537653 25536 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [LEADER]: Connected to new peer: Peer: permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
10:27:35.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.556324 25562 sys_catalog.cc:455] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1242297439664bee887da43b847e2418. Latest consensus state: current_term: 2 leader_uuid: "1242297439664bee887da43b847e2418" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:35.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.557256 25562 sys_catalog.cc:458] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: This master's current role is: FOLLOWER
10:27:35.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.575637 25562 sys_catalog.cc:455] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "1242297439664bee887da43b847e2418" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:35.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.576867 25562 sys_catalog.cc:458] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: This master's current role is: FOLLOWER
10:27:35.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.587131 25536 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "1242297439664bee887da43b847e2418" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:35.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.588001 25536 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: This master's current role is: LEADER
10:27:35.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:35.590325 25386 consensus_peers.cc:487] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 -> Peer 563cf286b3ac4c269c5c6359def4bf91 (127.24.114.60:43033): Couldn't send request to peer 563cf286b3ac4c269c5c6359def4bf91. Status: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:27:36.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:35.361768 25559 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:36.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:35.361966 25526 server_base.cc:1034] running on GCE node
10:27:36.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:35.361451 25556 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:36.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:36.677258 25526 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:36.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:36.681736 25526 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:36.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:36.683177 25526 hybrid_clock.cc:648] HybridClock initialized: now 1736936856683129 us; error 59 us; skew 500 ppm
10:27:36.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:36.684324 25526 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:36.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:36 dist-test-slave-hht6 krb5kdc[25043](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936856, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:27:36.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:36.692245 25526 init.cc:377] Logged in from keytab as kudu/127.24.114.60@KRBTEST.COM (short username kudu)
10:27:36.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:36.698673 25526 webserver.cc:458] Webserver started at http://127.24.114.60:45607/ using document root <none> and password file <none>
10:27:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:36.699936 25526 fs_manager.cc:362] Metadata directory not provided
10:27:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:36.700225 25526 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:36.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:36.711129 25526 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.002s sys 0.003s
10:27:36.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:36.716812 25578 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:36.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:36.718051 25526 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
10:27:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:36.718437 25526 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data,/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/wal
10:27:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "563cf286b3ac4c269c5c6359def4bf91"
10:27:36.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:27 on dist-test-slave-hht6"
10:27:36.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:36.721820 25526 fs_report.cc:389] FS layout report
10:27:36.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:36.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/wal
10:27:36.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/wal
10:27:36.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data/data
10:27:36.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:36.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:36.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:36.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:36.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:36.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:36.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:36.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:36.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:36.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:36.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:36.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:36.790297 25526 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:27:36.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:36.798369 25526 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:36.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:36.798936 25526 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:36.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:36.929630 25526 rpc_server.cc:307] RPC server started. Bound to: 127.24.114.60:43033
10:27:36.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:36.929759 25630 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.114.60:43033 every 8 connection(s)
10:27:36.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:36.932871 25526 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data/info.pb
10:27:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:36.937975 25032 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu as pid 25526
10:27:36.938 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:27:36.939 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.24.114.61:42113(127.24.114.61:42113)] connecting to peer
10:27:36.941 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.24.114.62:41373(127.24.114.62:41373)] connecting to peer
10:27:36.942 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.24.114.60:43033(127.24.114.60:43033)] connecting to peer
10:27:36.946 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.24.114.61:42113(127.24.114.61:42113)] Successfully connected to peer
10:27:36.948 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.24.114.60:43033(127.24.114.60:43033)] Successfully connected to peer
10:27:36.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:36.948006 25631 sys_catalog.cc:263] Verifying existing consensus state
10:27:36.953 [DEBUG - kudu-nio-0] (SecurityContext.java:225) Refreshing Kerberos credentials...
10:27:36.953 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5cc
10:27:36.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:36 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936847, etypes {rep=17 tkt=17 ses=17}, test-user@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:27:36.960 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.24.114.62:41373(127.24.114.62:41373)] Successfully connected to peer
10:27:36.961 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-user@KRBTEST.COM
10:27:36.964 [ERROR - kudu-nio-0] (SecurityContext.java:249) Attempted to refresh Kerberos credentials from ticket cache but found that the new Kerberos principal test-user@KRBTEST.COM did not match the original principal test-admin@KRBTEST.COM. Ignoring.
10:27:36.965 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.24.114.61
10:27:36.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:36.967861 25631 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: Bootstrap starting.
10:27:36.980 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.24.114.62
10:27:36.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:36 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936847, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.114.61@KRBTEST.COM
10:27:37.001 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.24.114.60
10:27:37.015 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.114.62:41373(127.24.114.62:41373)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
10:27:37.016 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.114.62:41373: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
10:27:37.016 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.24.114.62:41373(127.24.114.62:41373)] handling channelInactive
10:27:37.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:37.019205 25638 negotiation.cc:337] Failed RPC negotiation. Trace:
10:27:37.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:36.952600 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:34728 (local address 127.24.114.62:41373)
10:27:37.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:36.972791 (+ 20191us) server_negotiation.cc:206] Beginning negotiation
10:27:37.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:36.972826 (+ 35us) server_negotiation.cc:399] Waiting for connection header
10:27:37.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:36.972880 (+ 54us) server_negotiation.cc:407] Connection header received
10:27:37.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:36.973069 (+ 189us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
10:27:37.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:36.973100 (+ 31us) server_negotiation.cc:461] Received NEGOTIATE request from client
10:27:37.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:36.973363 (+ 263us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
10:27:37.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:36.973588 (+ 225us) server_negotiation.cc:226] Negotiated authn=SASL
10:27:37.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:36.984625 (+ 11037us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
10:27:37.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:36.988985 (+ 4360us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
10:27:37.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:37.015368 (+ 26383us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
10:27:37.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:37.015992 (+ 624us) server_negotiation.cc:662] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
10:27:37.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:37.017322 (+ 1330us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:34728: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:34728): Cannot send after transport endpoint shutdown (error 108)
10:27:37.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":19013,"thread_start_us":19104,"threads_started":1}
10:27:37.021 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x3a28a0f2, L:/127.0.0.1:58986 - R:/127.24.114.61:42113] using SASL/GSSAPI
10:27:37.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:37 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936852, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.61@KRBTEST.COM for kudu/127.24.114.60@KRBTEST.COM
10:27:37.039 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
10:27:37.040 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=101), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.114.61:42113, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.114.62:41373, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.114.60:43033, [77ms] Sub RPC ConnectToMaster: received response from server master-127.24.114.62:41373: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [96ms] Sub RPC ConnectToMaster: received response from server master-127.24.114.61:42113: OK, deferred=null) after lookup
10:27:37.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:37.050954 25631 log.cc:826] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: Log is configured to *not* fsync() on all Append() calls
10:27:37.061 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.114.60:43033(127.24.114.60:43033)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
10:27:37.062 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.114.60:43033: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
10:27:37.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:37.065675 25637 negotiation.cc:337] Failed RPC negotiation. Trace:
10:27:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:36.948688 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:47636 (local address 127.24.114.60:43033)
10:27:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:36.961013 (+ 12325us) server_negotiation.cc:206] Beginning negotiation
10:27:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:36.961050 (+ 37us) server_negotiation.cc:399] Waiting for connection header
10:27:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:36.961109 (+ 59us) server_negotiation.cc:407] Connection header received
10:27:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:36.977655 (+ 16546us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
10:27:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:36.977715 (+ 60us) server_negotiation.cc:461] Received NEGOTIATE request from client
10:27:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:36.978365 (+ 650us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
10:27:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:36.978828 (+ 463us) server_negotiation.cc:226] Negotiated authn=SASL
10:27:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:37.017508 (+ 38680us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
10:27:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:37.019938 (+ 2430us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
10:27:37.067 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.24.114.60:43033(127.24.114.60:43033)] handling channelInactive
10:27:37.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:37.059846 (+ 39908us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
10:27:37.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:37.060505 (+ 659us) server_negotiation.cc:662] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
10:27:37.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:37.063570 (+ 3065us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:47636: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:47636): Cannot send after transport endpoint shutdown (error 108)
10:27:37.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":11722,"thread_start_us":11278,"threads_started":1}
10:27:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:37.075788 25631 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:27:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:37.076599 25631 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: Bootstrap complete.
10:27:37.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:37.101531 25631 raft_consensus.cc:357] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:37.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:37.104049 25631 raft_consensus.cc:738] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 563cf286b3ac4c269c5c6359def4bf91, State: Initialized, Role: FOLLOWER
10:27:37.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:37.105089 25631 consensus_queue.cc:260] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:37.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:37.108922 25631 sys_catalog.cc:564] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: configured and running, proceeding with master startup.
10:27:37.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:37.109622 25605 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [term 1 FOLLOWER]: Advancing to term 2
10:27:37.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:37.108110 25641 sys_catalog.cc:455] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:37.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:37.110589 25641 sys_catalog.cc:458] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: This master's current role is: FOLLOWER
10:27:37.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:37.131421 25651 sys_catalog.cc:455] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "1242297439664bee887da43b847e2418" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:37.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:37.132866 25651 sys_catalog.cc:458] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: This master's current role is: FOLLOWER
10:27:37.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:37.138120 25641 sys_catalog.cc:455] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1242297439664bee887da43b847e2418. Latest consensus state: current_term: 2 leader_uuid: "1242297439664bee887da43b847e2418" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:37.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:37.138828 25641 sys_catalog.cc:458] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: This master's current role is: FOLLOWER
10:27:37.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:37 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936856, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.60@KRBTEST.COM for kudu/127.24.114.62@KRBTEST.COM
10:27:37.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:37.165500 25655 catalog_manager.cc:1260] Loaded cluster ID: 3f87294c829d403599dd52ce54e7039e
10:27:37.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:37.165819 25655 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: loading cluster ID for follower catalog manager: success
10:27:37.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:37.173833 25655 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: acquiring CA information for follower catalog manager: success
10:27:37.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:37.179581 25655 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:27:37.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:37.418224 25568 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "1242297439664bee887da43b847e2418" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:37.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:37.419026 25568 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: This master's current role is: LEADER
10:27:38.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:38.217388 24377 debug-util.cc:398] Leaking SignalData structure 0x7b0800084a20 after lost signal to thread 24314
10:27:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:38.218288 24377 debug-util.cc:398] Leaking SignalData structure 0x7b08000a0e20 after lost signal to thread 24380
10:27:40.119 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.114.61:42113
10:27:40.120 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
10:27:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:40.120465 25032 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu with pid 25367
10:27:40.147 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.24.114.61:42113(127.24.114.61:42113)] handling channelInactive
10:27:40.147 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.114.61:42113(127.24.114.61:42113)] cleaning up while in state READY due to: connection closed
10:27:40.149 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:27:40.149 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.114.62:41373
10:27:40.149 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
10:27:40.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:40.150040 25032 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu with pid 25439
10:27:40.179 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:27:40.179 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.114.60:43033
10:27:40.179 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
10:27:40.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:40.179590 25032 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu with pid 25526
10:27:40.206 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:27:40.206 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.114.61:42113
10:27:40.206 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
10:27:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:40.208077 25032 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/wal
10:27:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data
10:27:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/logs
10:27:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data/info.pb
10:27:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:40.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.61:42113
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.61
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42711
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.61.keytab
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.61
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:40.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
10:27:40.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:40.523531 25669 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:40.524070 25669 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:40.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:40.524789 25669 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:40.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:40.559019 25669 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
10:27:40.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:40.559285 25669 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:27:40.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:40.559696 25669 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:40.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:40.559979 25669 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:27:40.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:40.560245 25669 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:27:40.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:40.560505 25669 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:27:40.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:40.560719 25669 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44767/jwks.json
10:27:40.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:40.600244 25669 master_runner.cc:386] Master server non-default flags:
10:27:40.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:40.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:40.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:40.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:40.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data
10:27:40.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/wal
10:27:40.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
10:27:40.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.61.keytab
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.61
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.61:42113
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data/info.pb
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.61
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42711
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/logs
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:27:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:27:40.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:40.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:40.601931 25669 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:40.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:40.603179 25669 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:27:40.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:40.618911 25675 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:41.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:40.619704 25678 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:41.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:40.619819 25676 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:41.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:40.620157 25669 server_base.cc:1034] running on GCE node
10:27:41.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:41.823694 25669 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:41.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:41.826388 25669 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:41.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:41.827745 25669 hybrid_clock.cc:648] HybridClock initialized: now 1736936861827695 us; error 63 us; skew 500 ppm
10:27:41.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:41.828553 25669 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:41.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:41 dist-test-slave-hht6 krb5kdc[25043](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936861, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:27:41.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:41.838313 25669 init.cc:377] Logged in from keytab as kudu/127.24.114.61@KRBTEST.COM (short username kudu)
10:27:41.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:41.843497 25669 webserver.cc:458] Webserver started at http://127.24.114.61:42711/ using document root <none> and password file <none>
10:27:41.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:41.844372 25669 fs_manager.cc:362] Metadata directory not provided
10:27:41.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:41.844566 25669 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:41.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:41.852569 25669 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.006s sys 0.001s
10:27:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:41.856993 25689 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:41.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:41.858064 25669 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
10:27:41.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:41.858333 25669 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data,/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/wal
10:27:41.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1242297439664bee887da43b847e2418"
10:27:41.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:25 on dist-test-slave-hht6"
10:27:41.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:41.860654 25669 fs_report.cc:389] FS layout report
10:27:41.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:41.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/wal
10:27:41.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/wal
10:27:41.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data/data
10:27:41.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:41.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:41.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:41.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:41.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:41.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:41.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:41.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:41.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:41.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:41.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:41.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:41.917974 25669 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:27:41.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:41.923172 25669 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:41.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:41.923610 25669 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:41.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:41.993718 25669 rpc_server.cc:307] RPC server started. Bound to: 127.24.114.61:42113
10:27:41.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:41.993834 25740 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.114.61:42113 every 8 connection(s)
10:27:41.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:41.996336 25669 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data/info.pb
10:27:42.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:42.000743 25032 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu as pid 25669
10:27:42.001 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:27:42.001 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.114.62:41373
10:27:42.001 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
10:27:42.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:42.003400 25032 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:42.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:42.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/wal
10:27:42.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data
10:27:42.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:42.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:42.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/logs
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data/info.pb
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.62:41373
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.62
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39231
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.62.keytab
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.62
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
10:27:42.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:42.008180 25741 sys_catalog.cc:263] Verifying existing consensus state
10:27:42.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:42.017462 25741 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: Bootstrap starting.
10:27:42.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:42.068233 25741 log.cc:826] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: Log is configured to *not* fsync() on all Append() calls
10:27:42.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:42.088088 25741 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:27:42.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:42.089159 25741 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: Bootstrap complete.
10:27:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:42.127835 25741 raft_consensus.cc:357] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:42.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:42.131637 25741 raft_consensus.cc:738] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1242297439664bee887da43b847e2418, State: Initialized, Role: FOLLOWER
10:27:42.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:42.132776 25741 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:42.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:42.135854 25745 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:42.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:42.137152 25745 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: This master's current role is: FOLLOWER
10:27:42.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:42.138207 25741 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: configured and running, proceeding with master startup.
10:27:42.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:42.174106 25756 catalog_manager.cc:1260] Loaded cluster ID: 3f87294c829d403599dd52ce54e7039e
10:27:42.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:42.174636 25756 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: loading cluster ID for follower catalog manager: success
10:27:42.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:42.183703 25756 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: acquiring CA information for follower catalog manager: success
10:27:42.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:42.189971 25756 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:27:42.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:42.170833 25669 master_runner.cc:186] Error getting master registration for 127.24.114.62:41373: Network error: Client connection negotiation failed: client connection to 127.24.114.62:41373: connect: Connection refused (error 111),
10:27:42.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:42.201889 25669 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:27:42.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:42.382128 25743 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:42.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:42.382750 25743 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:42.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:42.383422 25743 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:42.414999 25743 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
10:27:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:42.415258 25743 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:27:42.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:42.415633 25743 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:42.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:42.415906 25743 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:27:42.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:42.416215 25743 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:27:42.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:42.416499 25743 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:27:42.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:42.416729 25743 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44767/jwks.json
10:27:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:42.455010 25743 master_runner.cc:386] Master server non-default flags:
10:27:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data
10:27:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/wal
10:27:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
10:27:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.62.keytab
10:27:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.62
10:27:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.62:41373
10:27:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:42.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
10:27:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
10:27:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data/info.pb
10:27:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.62
10:27:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39231
10:27:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/logs
10:27:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:27:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:27:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:27:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:27:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:42.456243 25743 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:42.457265 25743 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:27:42.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:42.472558 25765 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:43.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:43.924551 25770 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:27:43.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:43.925161 25770 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:43.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:43.931157 25691 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.114.62:41373: connect: Connection refused (error 111)
10:27:43.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:43.946218 25770 leader_election.cc:290] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 584493ef8c844a048f121a878c0e4745 (127.24.114.62:41373), 563cf286b3ac4c269c5c6359def4bf91 (127.24.114.60:43033)
10:27:43.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:43.950302 25691 leader_election.cc:336] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 584493ef8c844a048f121a878c0e4745 (127.24.114.62:41373): Network error: Client connection negotiation failed: client connection to 127.24.114.62:41373: connect: Connection refused (error 111)
10:27:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:43.952745 25690 leader_election.cc:336] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 563cf286b3ac4c269c5c6359def4bf91 (127.24.114.60:43033): Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:43.953246 25690 leader_election.cc:304] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1242297439664bee887da43b847e2418; no voters: 563cf286b3ac4c269c5c6359def4bf91, 584493ef8c844a048f121a878c0e4745
10:27:43.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:43.954022 25770 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
10:27:44.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:42.473613 25764 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:44.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:42.475602 25767 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:44.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:44.074077 25766 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1596 milliseconds
10:27:44.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.074172 25743 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:27:44.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.075403 25743 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.078542 25743 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.079983 25743 hybrid_clock.cc:648] HybridClock initialized: now 1736936864079949 us; error 33 us; skew 500 ppm
10:27:44.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.080897 25743 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:44.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:44 dist-test-slave-hht6 krb5kdc[25043](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936864, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:27:44.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.087152 25743 init.cc:377] Logged in from keytab as kudu/127.24.114.62@KRBTEST.COM (short username kudu)
10:27:44.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.092819 25743 webserver.cc:458] Webserver started at http://127.24.114.62:39231/ using document root <none> and password file <none>
10:27:44.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.094120 25743 fs_manager.cc:362] Metadata directory not provided
10:27:44.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.094506 25743 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:44.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.105329 25743 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.007s sys 0.001s
10:27:44.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.110870 25779 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:44.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.112179 25743 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
10:27:44.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.112567 25743 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data,/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/wal
10:27:44.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "584493ef8c844a048f121a878c0e4745"
10:27:44.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:23 on dist-test-slave-hht6"
10:27:44.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.116004 25743 fs_report.cc:389] FS layout report
10:27:44.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:44.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/wal
10:27:44.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/wal
10:27:44.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data/data
10:27:44.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:44.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:44.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:44.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:44.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:44.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:44.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:44.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:44.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:44.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:44.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:44.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.201335 25743 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:27:44.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.207190 25743 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:44.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.207674 25743 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:44.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.379927 25743 rpc_server.cc:307] RPC server started. Bound to: 127.24.114.62:41373
10:27:44.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.380029 25830 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.114.62:41373 every 8 connection(s)
10:27:44.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.383991 25743 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data/info.pb
10:27:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.388219 25032 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu as pid 25743
10:27:44.388 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:27:44.389 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.114.60:43033
10:27:44.389 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.390865 25032 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/wal
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/logs
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data/info.pb
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.60:43033
10:27:44.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.60
10:27:44.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45607
10:27:44.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.60.keytab
10:27:44.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.60
10:27:44.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:44.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
10:27:44.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
10:27:44.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:44.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:44.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:44.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:44.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:44.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:44.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:44.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:44.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
10:27:44.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.396245 25831 sys_catalog.cc:263] Verifying existing consensus state
10:27:44.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.405110 25831 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: Bootstrap starting.
10:27:44.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.458899 25831 log.cc:826] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: Log is configured to *not* fsync() on all Append() calls
10:27:44.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.485734 25831 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:27:44.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.486763 25831 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: Bootstrap complete.
10:27:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.523593 25831 raft_consensus.cc:357] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.527252 25831 raft_consensus.cc:738] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 584493ef8c844a048f121a878c0e4745, State: Initialized, Role: FOLLOWER
10:27:44.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.528249 25831 consensus_queue.cc:260] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:44.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.531399 25835 sys_catalog.cc:455] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:44.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.532518 25835 sys_catalog.cc:458] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: This master's current role is: FOLLOWER
10:27:44.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.533288 25831 sys_catalog.cc:564] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: configured and running, proceeding with master startup.
10:27:44.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.594633 25846 catalog_manager.cc:1260] Loaded cluster ID: 3f87294c829d403599dd52ce54e7039e
10:27:44.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.594947 25846 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: loading cluster ID for follower catalog manager: success
10:27:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.603005 25846 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: acquiring CA information for follower catalog manager: success
10:27:44.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.609043 25846 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:27:44.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:44 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936864, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.62@KRBTEST.COM for kudu/127.24.114.62@KRBTEST.COM
10:27:44.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.696542 25743 master_runner.cc:186] Error getting master registration for 127.24.114.60:43033: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111),
10:27:44.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.732714 25743 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:27:44.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:44.815441 25833 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:44.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:44.816136 25833 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:44.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:44.817201 25833 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:44.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:44.852442 25833 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
10:27:44.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:44.852716 25833 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:27:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:44.853212 25833 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:44.853509 25833 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:27:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:44.853821 25833 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:27:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:44.854113 25833 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:27:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:44.854321 25833 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44767/jwks.json
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.921609 25833 master_runner.cc:386] Master server non-default flags:
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/wal
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.60.keytab
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.60
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.60:43033
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data/info.pb
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.60
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45607
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:44.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:44.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:44.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/logs
10:27:44.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:44.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:44.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:44.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:27:44.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:44.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:27:44.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:44.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:27:44.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:27:44.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:44.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.923444 25833 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:44.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:44.924751 25833 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:27:44.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:44.945943 25857 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:45.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:45.315322 25017 debug-util.cc:398] Leaking SignalData structure 0x7b08000b1060 after lost signal to thread 24888
10:27:45.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:45.316816 25017 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7660 after lost signal to thread 25020
10:27:45.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.708731 25861 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:27:45.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.710573 25861 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:45.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.739161 25861 leader_election.cc:290] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 584493ef8c844a048f121a878c0e4745 (127.24.114.62:41373), 563cf286b3ac4c269c5c6359def4bf91 (127.24.114.60:43033)
10:27:45.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.751629 25806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1242297439664bee887da43b847e2418" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "584493ef8c844a048f121a878c0e4745" is_pre_election: true
10:27:45.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.752676 25806 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1242297439664bee887da43b847e2418 in term 2.
10:27:45.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.756011 25691 leader_election.cc:304] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1242297439664bee887da43b847e2418, 584493ef8c844a048f121a878c0e4745; no voters:
10:27:45.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.756914 25861 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 2 FOLLOWER]: Leader pre-election won for term 3
10:27:45.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.757613 25861 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
10:27:45.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.758276 25861 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 2 FOLLOWER]: Advancing to term 3
10:27:45.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:45.778105 25690 leader_election.cc:336] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 563cf286b3ac4c269c5c6359def4bf91 (127.24.114.60:43033): Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:45.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.780915 25861 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:45.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.784909 25806 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1242297439664bee887da43b847e2418" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "584493ef8c844a048f121a878c0e4745"
10:27:45.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.785578 25806 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 2 FOLLOWER]: Advancing to term 3
10:27:45.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.802772 25806 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1242297439664bee887da43b847e2418 in term 3.
10:27:45.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.804203 25691 leader_election.cc:304] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1242297439664bee887da43b847e2418, 584493ef8c844a048f121a878c0e4745; no voters:
10:27:45.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.817942 25866 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 3 FOLLOWER]: Leader election won for term 3
10:27:45.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:45.823177 25690 leader_election.cc:336] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 563cf286b3ac4c269c5c6359def4bf91 (127.24.114.60:43033): Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:45.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.823992 25861 leader_election.cc:290] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [CANDIDATE]: Term 3 election: Requested vote from peers 584493ef8c844a048f121a878c0e4745 (127.24.114.62:41373), 563cf286b3ac4c269c5c6359def4bf91 (127.24.114.60:43033)
10:27:45.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.829072 25866 raft_consensus.cc:695] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 3 LEADER]: Becoming Leader. State: Replica: 1242297439664bee887da43b847e2418, State: Running, Role: LEADER
10:27:45.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.830212 25866 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:45.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.853899 25861 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1242297439664bee887da43b847e2418. Latest consensus state: current_term: 3 leader_uuid: "1242297439664bee887da43b847e2418" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:45.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.855731 25861 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: This master's current role is: LEADER
10:27:45.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.886513 25869 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
10:27:45.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.951543 25869 catalog_manager.cc:1485] Initializing Kudu cluster ID...
10:27:45.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.964856 25869 catalog_manager.cc:1260] Loaded cluster ID: 3f87294c829d403599dd52ce54e7039e
10:27:45.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.970237 25869 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
10:27:45.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.977967 25869 catalog_manager.cc:1505] Loading token signing keys...
10:27:45.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:45.997526 25869 catalog_manager.cc:5910] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: Loaded TSK: 0
10:27:46.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.003319 25869 catalog_manager.cc:1515] Initializing in-progress tserver states...
10:27:46.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.224428 25806 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 3 FOLLOWER]: Refusing update from remote peer 1242297439664bee887da43b847e2418: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
10:27:46.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.226527 25861 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [LEADER]: Connected to new peer: Peer: permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
10:27:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:46.239640 25690 consensus_peers.cc:487] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 -> Peer 563cf286b3ac4c269c5c6359def4bf91 (127.24.114.60:43033): Couldn't send request to peer 563cf286b3ac4c269c5c6359def4bf91. Status: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:27:46.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.257885 25870 sys_catalog.cc:455] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1242297439664bee887da43b847e2418. Latest consensus state: current_term: 3 leader_uuid: "1242297439664bee887da43b847e2418" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:46.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.258859 25870 sys_catalog.cc:458] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: This master's current role is: FOLLOWER
10:27:46.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.269695 25870 sys_catalog.cc:455] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "1242297439664bee887da43b847e2418" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:46.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.270718 25870 sys_catalog.cc:458] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: This master's current role is: FOLLOWER
10:27:46.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.277559 25861 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "1242297439664bee887da43b847e2418" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:46.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.278276 25861 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: This master's current role is: LEADER
10:27:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:46.434542 24531 debug-util.cc:398] Leaking SignalData structure 0x7b08000bf920 after lost signal to thread 24463
10:27:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:46.435396 24531 debug-util.cc:398] Leaking SignalData structure 0x7b080008c7c0 after lost signal to thread 24534
10:27:46.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:46.341303 25856 debug-util.cc:398] Leaking SignalData structure 0x7b080001cac0 after lost signal to thread 25833
10:27:46.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:46.701047 25856 kernel_stack_watchdog.cc:198] Thread 25833 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:636 for 395ms:
10:27:46.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
10:27:46.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
10:27:46.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
10:27:46.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
10:27:46.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:44.945957 25858 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:46.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:46.702457 25833 thread.cc:635] GCE (cloud detector) Time spent creating pthread: real 1.759s user 0.456s sys 0.874s
10:27:46.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:46.702842 25833 thread.cc:601] GCE (cloud detector) Time spent starting thread: real 1.759s user 0.456s sys 0.874s
10:27:46.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:46.715263 25876 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:46.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.716255 25833 server_base.cc:1034] running on GCE node
10:27:46.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.717794 25833 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:46.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.733144 25833 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:46.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.736713 25833 hybrid_clock.cc:648] HybridClock initialized: now 1736936866736663 us; error 47 us; skew 500 ppm
10:27:46.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.737900 25833 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:46.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:46 dist-test-slave-hht6 krb5kdc[25043](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936866, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:27:46.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.745875 25833 init.cc:377] Logged in from keytab as kudu/127.24.114.60@KRBTEST.COM (short username kudu)
10:27:46.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.789857 25833 webserver.cc:458] Webserver started at http://127.24.114.60:45607/ using document root <none> and password file <none>
10:27:46.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.791412 25833 fs_manager.cc:362] Metadata directory not provided
10:27:46.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.791839 25833 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:46.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.812927 25833 fs_manager.cc:714] Time spent opening directory manager: real 0.012s user 0.013s sys 0.001s
10:27:46.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.824666 25882 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:46.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.826238 25833 fs_manager.cc:730] Time spent opening block manager: real 0.010s user 0.008s sys 0.000s
10:27:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.826885 25833 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data,/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/wal
10:27:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "563cf286b3ac4c269c5c6359def4bf91"
10:27:46.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:27 on dist-test-slave-hht6"
10:27:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.830897 25833 fs_report.cc:389] FS layout report
10:27:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/wal
10:27:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/wal
10:27:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data/data
10:27:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:46.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.932379 25833 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:27:46.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.960129 25833 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:46.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:46.961292 25833 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:47.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:47.248340 25669 master_runner.cc:186] Error getting master registration for 127.24.114.60:43033: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111),
10:27:47.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:47.248829 25669 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:27:47.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:47.417321 25937 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.114.60:43033 every 8 connection(s)
10:27:47.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:47.418054 25833 rpc_server.cc:307] RPC server started. Bound to: 127.24.114.60:43033
10:27:47.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:47.428272 25833 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data/info.pb
10:27:47.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:47.436779 25032 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu as pid 25833
10:27:47.437 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:27:47.438 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.24.114.61:42113(127.24.114.61:42113)] connecting to peer
10:27:47.442 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.24.114.61:42113(127.24.114.61:42113)] Successfully connected to peer
10:27:47.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:47.469524 25938 sys_catalog.cc:263] Verifying existing consensus state
10:27:47.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:47.477924 25938 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: Bootstrap starting.
10:27:47.493 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x87dd891d, L:/127.0.0.1:44954 - R:/127.24.114.61:42113] using TOKEN/null
10:27:47.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:47.531122 25938 log.cc:826] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: Log is configured to *not* fsync() on all Append() calls
10:27:47.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:47.569669 25938 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:27:47.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:47.570935 25938 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: Bootstrap complete.
10:27:47.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:47.616079 25938 raft_consensus.cc:357] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:47.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:47.620095 25938 raft_consensus.cc:738] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 563cf286b3ac4c269c5c6359def4bf91, State: Initialized, Role: FOLLOWER
10:27:47.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:47.621577 25938 consensus_queue.cc:260] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:47.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:47.630522 25941 sys_catalog.cc:455] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:47.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:47.634824 25941 sys_catalog.cc:458] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: This master's current role is: FOLLOWER
10:27:47.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:47.634449 25938 sys_catalog.cc:564] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: configured and running, proceeding with master startup.
10:27:47.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:47.725194 25952 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
10:27:47.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:47 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936866, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.60@KRBTEST.COM for kudu/127.24.114.62@KRBTEST.COM
10:27:47.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:47 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936861, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.61@KRBTEST.COM for kudu/127.24.114.60@KRBTEST.COM
10:27:47.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:47 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936866, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.60@KRBTEST.COM for kudu/127.24.114.61@KRBTEST.COM
10:27:47.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:47.836653 25911 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [term 2 FOLLOWER]: Advancing to term 3
10:27:47.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:47 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936866, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.60@KRBTEST.COM for kudu/127.24.114.60@KRBTEST.COM
10:27:47.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:47.967020 25941 sys_catalog.cc:455] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1242297439664bee887da43b847e2418. Latest consensus state: current_term: 3 leader_uuid: "1242297439664bee887da43b847e2418" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:47.967895 25941 sys_catalog.cc:458] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: This master's current role is: FOLLOWER
10:27:47.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:47.978660 25959 sys_catalog.cc:455] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "1242297439664bee887da43b847e2418" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:47.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:47.979730 25959 sys_catalog.cc:458] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: This master's current role is: FOLLOWER
10:27:48.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:48.148001 25866 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "1242297439664bee887da43b847e2418" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:48.149943 25866 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: This master's current role is: LEADER
10:27:48.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:48.747972 25952 catalog_manager.cc:1260] Loaded cluster ID: 3f87294c829d403599dd52ce54e7039e
10:27:48.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:48.748282 25952 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: loading cluster ID for follower catalog manager: success
10:27:48.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:48.755947 25952 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: acquiring CA information for follower catalog manager: success
10:27:48.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:48.762069 25952 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:27:50.558 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.114.61:42113
10:27:50.558 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
10:27:50.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:50.558696 25032 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu with pid 25669
10:27:50.582 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:27:50.582 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.114.62:41373
10:27:50.582 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.24.114.61:42113(127.24.114.61:42113)] handling channelInactive
10:27:50.582 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
10:27:50.582 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.114.61:42113(127.24.114.61:42113)] cleaning up while in state READY due to: connection closed
10:27:50.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:50.582867 25032 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu with pid 25743
10:27:50.611 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:27:50.611 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.114.60:43033
10:27:50.611 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
10:27:50.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:50.611956 25032 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu with pid 25833
10:27:50.639 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:27:50.640 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.114.61:42113
10:27:50.640 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
10:27:50.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:50.641595 25032 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/wal
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/logs
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data/info.pb
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.61:42113
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.61
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42711
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.61.keytab
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.61
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:50.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:50.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:50.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:50.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
10:27:50.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:50.959347 25969 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:50.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:50.959956 25969 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:50.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:50.960652 25969 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:50.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:50.994522 25969 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
10:27:50.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:50.994827 25969 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:27:50.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:50.995124 25969 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:50.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:50.995335 25969 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:27:50.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:50.995522 25969 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:27:50.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:50.995702 25969 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:27:50.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:50.995846 25969 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44767/jwks.json
10:27:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:51.035059 25969 master_runner.cc:386] Master server non-default flags:
10:27:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data
10:27:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/wal
10:27:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
10:27:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.61.keytab
10:27:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.61
10:27:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.61:42113
10:27:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
10:27:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
10:27:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data/info.pb
10:27:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.61
10:27:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42711
10:27:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/logs
10:27:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:27:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:27:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:27:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:27:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:51.036461 25969 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:51.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:51.037451 25969 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:27:51.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:51.053418 25976 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:52.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:51.054119 25977 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:52.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:51.053962 25979 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:52.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:52.265502 25978 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
10:27:52.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.265594 25969 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:27:52.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.269688 25969 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:52.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.272109 25969 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:52.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.273423 25969 hybrid_clock.cc:648] HybridClock initialized: now 1736936872273387 us; error 52 us; skew 500 ppm
10:27:52.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.274215 25969 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:52 dist-test-slave-hht6 krb5kdc[25043](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936872, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:27:52.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.284780 25969 init.cc:377] Logged in from keytab as kudu/127.24.114.61@KRBTEST.COM (short username kudu)
10:27:52.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.289520 25969 webserver.cc:458] Webserver started at http://127.24.114.61:42711/ using document root <none> and password file <none>
10:27:52.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.290396 25969 fs_manager.cc:362] Metadata directory not provided
10:27:52.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.290578 25969 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:52.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.298386 25969 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.001s sys 0.004s
10:27:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.302675 25988 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:52.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.303891 25969 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
10:27:52.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.304208 25969 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data,/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/wal
10:27:52.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1242297439664bee887da43b847e2418"
10:27:52.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:25 on dist-test-slave-hht6"
10:27:52.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.306596 25969 fs_report.cc:389] FS layout report
10:27:52.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:52.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/wal
10:27:52.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/wal
10:27:52.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data/data
10:27:52.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:52.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:52.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:52.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:52.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:52.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:52.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:52.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:52.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:52.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:52.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:52.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.365270 25969 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:27:52.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.370512 25969 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:52.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.370936 25969 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:52.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.440932 25969 rpc_server.cc:307] RPC server started. Bound to: 127.24.114.61:42113
10:27:52.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.441023 26039 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.114.61:42113 every 8 connection(s)
10:27:52.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.443775 25969 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-1/data/info.pb
10:27:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.453330 25032 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu as pid 25969
10:27:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.453611 26040 sys_catalog.cc:263] Verifying existing consensus state
10:27:52.454 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:27:52.454 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.114.62:41373
10:27:52.454 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
10:27:52.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.455838 25032 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:52.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:52.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/wal
10:27:52.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data
10:27:52.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:52.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:52.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/logs
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data/info.pb
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.62:41373
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.62
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39231
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.62.keytab
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.62
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:52.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
10:27:52.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.459581 26040 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: Bootstrap starting.
10:27:52.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.512317 26040 log.cc:826] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: Log is configured to *not* fsync() on all Append() calls
10:27:52.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.534431 26040 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:27:52.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.535457 26040 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: Bootstrap complete.
10:27:52.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.571202 26040 raft_consensus.cc:357] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:52.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.574838 26040 raft_consensus.cc:738] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1242297439664bee887da43b847e2418, State: Initialized, Role: FOLLOWER
10:27:52.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.575882 26040 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:52.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.578954 26044 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:52.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.579674 26044 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: This master's current role is: FOLLOWER
10:27:52.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.581286 26040 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [sys.catalog]: configured and running, proceeding with master startup.
10:27:52.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.614460 26055 catalog_manager.cc:1260] Loaded cluster ID: 3f87294c829d403599dd52ce54e7039e
10:27:52.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.615734 26055 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: loading cluster ID for follower catalog manager: success
10:27:52.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.624166 26055 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: acquiring CA information for follower catalog manager: success
10:27:52.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.630389 26055 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:27:52.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.615618 25969 master_runner.cc:186] Error getting master registration for 127.24.114.62:41373: Network error: Client connection negotiation failed: client connection to 127.24.114.62:41373: connect: Connection refused (error 111),
10:27:52.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.642554 25969 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:27:52.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:52.835953 26042 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:52.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:52.836714 26042 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:52.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:52.837419 26042 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:52.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:52.872045 26042 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
10:27:52.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:52.872346 26042 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:27:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:52.872817 26042 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:52.873165 26042 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:27:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:52.873479 26042 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:27:52.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:52.873792 26042 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:27:52.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:52.874023 26042 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44767/jwks.json
10:27:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.939052 26042 master_runner.cc:386] Master server non-default flags:
10:27:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data
10:27:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/wal
10:27:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
10:27:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.62.keytab
10:27:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.62
10:27:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.62:41373
10:27:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
10:27:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
10:27:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data/info.pb
10:27:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.62
10:27:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39231
10:27:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/logs
10:27:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:27:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:27:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:27:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:27:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.940721 26042 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.941992 26042 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:27:52.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:52.959661 26064 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:54.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.143025 26070 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:27:54.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.143719 26070 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:54.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.148192 26070 leader_election.cc:290] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 584493ef8c844a048f121a878c0e4745 (127.24.114.62:41373), 563cf286b3ac4c269c5c6359def4bf91 (127.24.114.60:43033)
10:27:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:54.162703 25990 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.114.62:41373: connect: Connection refused (error 111)
10:27:54.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:54.168359 25990 leader_election.cc:336] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 584493ef8c844a048f121a878c0e4745 (127.24.114.62:41373): Network error: Client connection negotiation failed: client connection to 127.24.114.62:41373: connect: Connection refused (error 111)
10:27:54.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:54.169188 25989 leader_election.cc:336] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 563cf286b3ac4c269c5c6359def4bf91 (127.24.114.60:43033): Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:54.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.169623 25989 leader_election.cc:304] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1242297439664bee887da43b847e2418; no voters: 563cf286b3ac4c269c5c6359def4bf91, 584493ef8c844a048f121a878c0e4745
10:27:54.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.170514 26070 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
10:27:54.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:52.961310 26063 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:54.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:52.962903 26042 server_base.cc:1034] running on GCE node
10:27:54.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:52.960541 26066 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:54.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.262889 26042 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:54.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.266110 26042 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:54.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.267539 26042 hybrid_clock.cc:648] HybridClock initialized: now 1736936874267480 us; error 76 us; skew 500 ppm
10:27:54.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.268324 26042 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:54.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:54 dist-test-slave-hht6 krb5kdc[25043](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936874, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:27:54.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.274431 26042 init.cc:377] Logged in from keytab as kudu/127.24.114.62@KRBTEST.COM (short username kudu)
10:27:54.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.280884 26042 webserver.cc:458] Webserver started at http://127.24.114.62:39231/ using document root <none> and password file <none>
10:27:54.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.281869 26042 fs_manager.cc:362] Metadata directory not provided
10:27:54.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.282096 26042 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:54.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.290714 26042 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
10:27:54.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.295836 26079 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:54.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.296928 26042 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.004s
10:27:54.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.297292 26042 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data,/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/wal
10:27:54.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "584493ef8c844a048f121a878c0e4745"
10:27:54.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:23 on dist-test-slave-hht6"
10:27:54.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.299551 26042 fs_report.cc:389] FS layout report
10:27:54.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:54.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/wal
10:27:54.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/wal
10:27:54.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data/data
10:27:54.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:54.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:54.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:54.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:54.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:54.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:54.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:54.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:54.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:54.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:54.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.372960 26042 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:27:54.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.378680 26042 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.379088 26042 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:54.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.470135 26042 rpc_server.cc:307] RPC server started. Bound to: 127.24.114.62:41373
10:27:54.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.470270 26130 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.114.62:41373 every 8 connection(s)
10:27:54.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.473074 26042 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-0/data/info.pb
10:27:54.477 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:27:54.477 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.114.60:43033
10:27:54.477 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
10:27:54.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.476848 25032 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu as pid 26042
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.479543 25032 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/wal
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/logs
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data/info.pb
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.60:43033
10:27:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.60
10:27:54.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45607
10:27:54.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.60.keytab
10:27:54.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.60
10:27:54.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:54.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
10:27:54.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
10:27:54.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:54.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:54.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:54.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:54.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:54.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:54.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:54.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:54.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
10:27:54.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.493139 26131 sys_catalog.cc:263] Verifying existing consensus state
10:27:54.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.501600 26131 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: Bootstrap starting.
10:27:54.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.553845 26131 log.cc:826] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: Log is configured to *not* fsync() on all Append() calls
10:27:54.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.575228 26131 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:27:54.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.575940 26131 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: Bootstrap complete.
10:27:54.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.599362 26131 raft_consensus.cc:357] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:54.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.601789 26131 raft_consensus.cc:738] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 584493ef8c844a048f121a878c0e4745, State: Initialized, Role: FOLLOWER
10:27:54.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.602488 26131 consensus_queue.cc:260] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:54.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.605532 26135 sys_catalog.cc:455] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:54.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.606415 26135 sys_catalog.cc:458] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: This master's current role is: FOLLOWER
10:27:54.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.607347 26131 sys_catalog.cc:564] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: configured and running, proceeding with master startup.
10:27:54.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.646277 26146 catalog_manager.cc:1260] Loaded cluster ID: 3f87294c829d403599dd52ce54e7039e
10:27:54.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.646579 26146 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: loading cluster ID for follower catalog manager: success
10:27:54.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.654938 26146 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: acquiring CA information for follower catalog manager: success
10:27:54.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:54 dist-test-slave-hht6 krb5kdc[25043](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936874, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.62@KRBTEST.COM for kudu/127.24.114.62@KRBTEST.COM
10:27:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.661999 26146 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
10:27:54.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.702333 26042 master_runner.cc:186] Error getting master registration for 127.24.114.60:43033: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111),
10:27:54.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.735352 26042 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
10:27:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:54.859664 26133 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
10:27:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:54.860220 26133 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
10:27:54.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:54.860915 26133 flags.cc:425] Enabled unsafe flag: --never_fsync=true
10:27:54.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:54.914371 26133 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
10:27:54.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:54.914692 26133 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
10:27:54.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:54.915163 26133 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
10:27:54.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:54.915489 26133 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
10:27:54.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:54.915802 26133 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
10:27:54.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:54.916100 26133 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
10:27:54.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:54.916316 26133 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44767/jwks.json
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.959734 26133 master_runner.cc:386] Master server non-default flags:
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.114.20:36287
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/wal
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.114.62:41373,127.24.114.61:42113,127.24.114.60:43033
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/kudu_127.24.114.60.keytab
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.114.60
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.114.60:43033
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44767/jwks.json
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data/info.pb
10:27:54.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.114.60
10:27:54.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45607
10:27:54.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
10:27:54.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
10:27:54.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
10:27:54.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
10:27:54.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
10:27:54.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/logs
10:27:54.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
10:27:54.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
10:27:54.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
10:27:54.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
10:27:54.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
10:27:54.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 75b75ca2f58f1a5aba69207830e3558011033d99
10:27:54.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
10:27:54.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 15 Jan 2025 10:06:40 UTC on a633016ff5c9
10:27:54.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4188
10:27:54.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
10:27:54.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.961090 26133 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
10:27:54.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:54.962255 26133 file_cache.cc:492] Constructed file cache file cache with capacity 419430
10:27:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:54.980162 26157 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:56.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.066295 26162 raft_consensus.cc:491] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
10:27:56.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.066896 26162 raft_consensus.cc:513] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.069813 26162 leader_election.cc:290] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 1242297439664bee887da43b847e2418 (127.24.114.61:42113), 563cf286b3ac4c269c5c6359def4bf91 (127.24.114.60:43033)
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.070922 26015 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "584493ef8c844a048f121a878c0e4745" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "1242297439664bee887da43b847e2418" is_pre_election: true
10:27:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.071695 26015 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 584493ef8c844a048f121a878c0e4745 in term 3.
10:27:56.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.073827 26082 leader_election.cc:304] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1242297439664bee887da43b847e2418, 584493ef8c844a048f121a878c0e4745; no voters:
10:27:56.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.074693 26162 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 3 FOLLOWER]: Leader pre-election won for term 4
10:27:56.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.075160 26162 raft_consensus.cc:491] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
10:27:56.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.075680 26162 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 3 FOLLOWER]: Advancing to term 4
10:27:56.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.085778 26162 raft_consensus.cc:513] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:56.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.087796 26162 leader_election.cc:290] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [CANDIDATE]: Term 4 election: Requested vote from peers 1242297439664bee887da43b847e2418 (127.24.114.61:42113), 563cf286b3ac4c269c5c6359def4bf91 (127.24.114.60:43033)
10:27:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.088564 26015 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "584493ef8c844a048f121a878c0e4745" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "1242297439664bee887da43b847e2418"
10:27:56.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.089170 26015 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 3 FOLLOWER]: Advancing to term 4
10:27:56.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.098513 26015 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 1242297439664bee887da43b847e2418 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 584493ef8c844a048f121a878c0e4745 in term 4.
10:27:56.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:56.098673 26080 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:56.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.099596 26082 leader_election.cc:304] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1242297439664bee887da43b847e2418, 584493ef8c844a048f121a878c0e4745; no voters:
10:27:56.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.100648 26162 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 4 FOLLOWER]: Leader election won for term 4
10:27:56.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:56.103811 26080 leader_election.cc:336] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 563cf286b3ac4c269c5c6359def4bf91 (127.24.114.60:43033): Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:56.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:56.106528 26080 leader_election.cc:336] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 563cf286b3ac4c269c5c6359def4bf91 (127.24.114.60:43033): Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111)
10:27:56.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.108848 26162 raft_consensus.cc:695] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [term 4 LEADER]: Becoming Leader. State: Replica: 584493ef8c844a048f121a878c0e4745, State: Running, Role: LEADER
10:27:56.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.109959 26162 consensus_queue.cc:237] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:56.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.127955 26165 sys_catalog.cc:455] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 584493ef8c844a048f121a878c0e4745. Latest consensus state: current_term: 4 leader_uuid: "584493ef8c844a048f121a878c0e4745" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:56.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.129088 26165 sys_catalog.cc:458] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 [sys.catalog]: This master's current role is: LEADER
10:27:56.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.140121 26167 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
10:27:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.150534 26167 catalog_manager.cc:1485] Initializing Kudu cluster ID...
10:27:56.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.155068 26167 catalog_manager.cc:1260] Loaded cluster ID: 3f87294c829d403599dd52ce54e7039e
10:27:56.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.155339 26167 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
10:27:56.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.160439 26167 catalog_manager.cc:1505] Loading token signing keys...
10:27:56.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.164812 26167 catalog_manager.cc:5910] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745: Loaded TSK: 0
10:27:56.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.166087 26167 catalog_manager.cc:1515] Initializing in-progress tserver states...
10:27:56.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:54.981366 26159 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:56.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:54.980535 26156 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
10:27:56.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:56.281556 26158 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
10:27:56.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.282146 26133 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
10:27:56.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.286255 26133 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
10:27:56.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.289077 26133 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
10:27:56.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.290450 26133 hybrid_clock.cc:648] HybridClock initialized: now 1736936876290404 us; error 55 us; skew 500 ppm
10:27:56.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.291290 26133 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
10:27:56.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 15 10:27:56 dist-test-slave-hht6 krb5kdc[25043](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736936876, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.114.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
10:27:56.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.297740 26133 init.cc:377] Logged in from keytab as kudu/127.24.114.60@KRBTEST.COM (short username kudu)
10:27:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.304282 26133 webserver.cc:458] Webserver started at http://127.24.114.60:45607/ using document root <none> and password file <none>
10:27:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.305248 26133 fs_manager.cc:362] Metadata directory not provided
10:27:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.305475 26133 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
10:27:56.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.314035 26133 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.007s sys 0.000s
10:27:56.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.319336 26173 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
10:27:56.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.320498 26133 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
10:27:56.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.320837 26133 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data,/tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/wal
10:27:56.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "563cf286b3ac4c269c5c6359def4bf91"
10:27:56.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-15 10:27:27 on dist-test-slave-hht6"
10:27:56.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.323199 26133 fs_report.cc:389] FS layout report
10:27:56.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
10:27:56.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/wal
10:27:56.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/wal
10:27:56.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data/data
10:27:56.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
10:27:56.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
10:27:56.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
10:27:56.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
10:27:56.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
10:27:56.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
10:27:56.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
10:27:56.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
10:27:56.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
10:27:56.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
10:27:56.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
10:27:56.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.382308 26133 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
10:27:56.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.387980 26133 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
10:27:56.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.388442 26133 kserver.cc:163] Server-wide thread pool size limit: 3276
10:27:56.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:56.507613 26080 consensus_peers.cc:487] T 00000000000000000000000000000000 P 584493ef8c844a048f121a878c0e4745 -> Peer 563cf286b3ac4c269c5c6359def4bf91 (127.24.114.60:43033): Couldn't send request to peer 563cf286b3ac4c269c5c6359def4bf91. Status: Network error: Client connection negotiation failed: client connection to 127.24.114.60:43033: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
10:27:56.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.534696 26133 rpc_server.cc:307] RPC server started. Bound to: 127.24.114.60:43033
10:27:56.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.534847 26225 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.114.60:43033 every 8 connection(s)
10:27:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.537953 26133 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/master-2/data/info.pb
10:27:56.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.542851 25032 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu as pid 26133
10:27:56.543 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
10:27:56.543 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.24.114.61:42113(127.24.114.61:42113)] connecting to peer
10:27:56.546 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.24.114.61:42113(127.24.114.61:42113)] Successfully connected to peer
10:27:56.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.551561 26226 sys_catalog.cc:263] Verifying existing consensus state
10:27:56.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.558144 26226 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: Bootstrap starting.
10:27:56.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:56.566397 26228 negotiation.cc:344] Unauthorized connection attempt: Server connection negotiation failed: server connection from 127.0.0.1:36130: token expired
10:27:56.574 [DEBUG - kudu-nio-0] (Negotiator.java:352) peer /127.24.114.61:42113 sent connection negotiation error: Not authorized: token expired
10:27:56.582 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.24.114.61:42113(127.24.114.61:42113)] handling channelInactive
10:27:56.583 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.114.61:42113(127.24.114.61:42113)] cleaning up while in state NEGOTIATION_FAILED due to: connection closed
10:27:56.586 [DEBUG - kudu-nio-0] (Connection.java:748) [peer master-127.24.114.61:42113(127.24.114.61:42113)] connecting to peer
10:27:56.589 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.24.114.61:42113(127.24.114.61:42113)] Successfully connected to peer
10:27:56.591 [DEBUG - kudu-nio-1] (SecurityContext.java:225) Refreshing Kerberos credentials...
10:27:56.592 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskaxjw05/test-tmp/mini-kudu-cluster3375483323158477800/krb5kdc/krb5cc
10:27:56.593 [DEBUG - kudu-nio-1] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
10:27:56.593 [WARN - kudu-nio-1] (SecurityContext.java:234) Tried to refresh Kerberos credentials but was unable to re-login from ticket cache
10:27:56.594 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.114.61:42113(127.24.114.61:42113)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
10:27:56.594 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.114.61:42113: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
10:27:56.594 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.24.114.61:42113(127.24.114.61:42113)] handling channelInactive
10:27:56.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:56.596882 26228 negotiation.cc:337] Failed RPC negotiation. Trace:
10:27:56.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.588380 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:36140 (local address 127.24.114.61:42113)
10:27:56.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.589141 (+ 761us) server_negotiation.cc:206] Beginning negotiation
10:27:56.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.589168 (+ 27us) server_negotiation.cc:399] Waiting for connection header
10:27:56.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.590441 (+ 1273us) server_negotiation.cc:407] Connection header received
10:27:56.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.590615 (+ 174us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
10:27:56.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.590654 (+ 39us) server_negotiation.cc:461] Received NEGOTIATE request from client
10:27:56.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.590965 (+ 311us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
10:27:56.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.591200 (+ 235us) server_negotiation.cc:226] Negotiated authn=SASL
10:27:56.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.595606 (+ 4406us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:36140: BlockingRecv error: recv got EOF from 127.0.0.1:36140 (error 108)
10:27:56.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":134}
10:27:56.600 [DEBUG - kudu-nio-0] (Connection.java:748) [peer master-127.24.114.62:41373(127.24.114.62:41373)] connecting to peer
10:27:56.601 [DEBUG - kudu-nio-0] (Connection.java:748) [peer master-127.24.114.60:43033(127.24.114.60:43033)] connecting to peer
10:27:56.603 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.24.114.60:43033(127.24.114.60:43033)] Successfully connected to peer
10:27:56.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.610999 26226 log.cc:826] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: Log is configured to *not* fsync() on all Append() calls
10:27:56.612 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.24.114.62:41373(127.24.114.62:41373)] Successfully connected to peer
10:27:56.622 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.114.62:41373(127.24.114.62:41373)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
10:27:56.623 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.114.62:41373: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
10:27:56.623 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.24.114.62:41373(127.24.114.62:41373)] handling channelInactive
10:27:56.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:56.625509 26230 negotiation.cc:337] Failed RPC negotiation. Trace:
10:27:56.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.606495 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:41446 (local address 127.24.114.62:41373)
10:27:56.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.621202 (+ 14707us) server_negotiation.cc:206] Beginning negotiation
10:27:56.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.621229 (+ 27us) server_negotiation.cc:399] Waiting for connection header
10:27:56.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.621280 (+ 51us) server_negotiation.cc:407] Connection header received
10:27:56.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.621430 (+ 150us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
10:27:56.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.621457 (+ 27us) server_negotiation.cc:461] Received NEGOTIATE request from client
10:27:56.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.621694 (+ 237us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
10:27:56.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.623492 (+ 1798us) server_negotiation.cc:226] Negotiated authn=SASL
10:27:56.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.624089 (+ 597us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:41446: BlockingRecv error: recv got EOF from 127.0.0.1:41446 (error 108)
10:27:56.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":14061,"thread_start_us":15381,"threads_started":1}
10:27:56.627 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.114.60:43033(127.24.114.60:43033)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
10:27:56.627 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.114.60:43033: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
10:27:56.628 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.24.114.60:43033(127.24.114.60:43033)] handling channelInactive
10:27:56.629 [DEBUG - main] (Connection.java:575) [peer master-127.24.114.62:41373(127.24.114.62:41373)] disconnecting while in state TERMINATED
10:27:56.629 [DEBUG - main] (Connection.java:575) [peer master-127.24.114.61:42113(127.24.114.61:42113)] disconnecting while in state TERMINATED
10:27:56.629 [DEBUG - main] (Connection.java:575) [peer master-127.24.114.60:43033(127.24.114.60:43033)] disconnecting while in state TERMINATED
10:27:56.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250115 10:27:56.630838 26229 negotiation.cc:337] Failed RPC negotiation. Trace:
10:27:56.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.603359 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:43562 (local address 127.24.114.60:43033)
10:27:56.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.618230 (+ 14871us) server_negotiation.cc:206] Beginning negotiation
10:27:56.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.618275 (+ 45us) server_negotiation.cc:399] Waiting for connection header
10:27:56.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.618351 (+ 76us) server_negotiation.cc:407] Connection header received
10:27:56.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.624973 (+ 6622us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
10:27:56.631 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
10:27:56.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.625025 (+ 52us) server_negotiation.cc:461] Received NEGOTIATE request from client
10:27:56.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.625474 (+ 449us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
10:27:56.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.627253 (+ 1779us) server_negotiation.cc:226] Negotiated authn=SASL
10:27:56.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0115 10:27:56.629184 (+ 1931us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:43562: BlockingRecv error: recv got EOF from 127.0.0.1:43562 (error 108)
10:27:56.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":13883,"thread_start_us":13328,"threads_started":1}
10:27:56.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.641472 26226 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
10:27:56.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.642766 25032 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu with pid 26042
10:27:56.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.643819 26226 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: Bootstrap complete.
10:27:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.666585 25032 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu with pid 25969
10:27:56.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.665462 26226 raft_consensus.cc:357] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:56.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.669036 26226 raft_consensus.cc:738] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 563cf286b3ac4c269c5c6359def4bf91, State: Initialized, Role: FOLLOWER
10:27:56.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.670029 26226 consensus_queue.cc:260] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } }
10:27:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.673254 26233 sys_catalog.cc:455] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "584493ef8c844a048f121a878c0e4745" member_type: VOTER last_known_addr { host: "127.24.114.62" port: 41373 } } peers { permanent_uuid: "1242297439664bee887da43b847e2418" member_type: VOTER last_known_addr { host: "127.24.114.61" port: 42113 } } peers { permanent_uuid: "563cf286b3ac4c269c5c6359def4bf91" member_type: VOTER last_known_addr { host: "127.24.114.60" port: 43033 } } }
10:27:56.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.674291 26233 sys_catalog.cc:458] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: This master's current role is: FOLLOWER
10:27:56.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.675241 26226 sys_catalog.cc:564] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91 [sys.catalog]: configured and running, proceeding with master startup.
10:27:56.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.706440 26244 catalog_manager.cc:1260] Loaded cluster ID: 3f87294c829d403599dd52ce54e7039e
10:27:56.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.706740 26244 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 563cf286b3ac4c269c5c6359def4bf91: loading cluster ID for follower catalog manager: success
10:27:56.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:56.707203 25032 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu with pid 26133
10:27:57.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-15T10:27:57Z chronyd exiting
10:27:58.933 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
10:27:58.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:58.934679 24301 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu with pid 24583
10:27:58.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:58.959256 24301 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu with pid 24746
10:27:58.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:58.984275 24301 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu with pid 24887
10:27:59.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:59.009665 24301 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu with pid 24313
10:27:59.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:59.039654 24301 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu with pid 24383
10:27:59.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250115 10:27:59.066848 24301 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskaxjw05/build/tsan/bin/kudu with pid 24458
10:28:00.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-15T10:28:00Z chronyd exiting
10:28:01.235 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.24.114.61:42113,127.24.114.62:41373,127.24.114.60:43033). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386) ~[test/:?]
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158) ~[main/:?]
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.24.114.61:42113,127.24.114.62:41373,127.24.114.60:43033). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158)
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more