03:07:25.487 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
03:07:25.498 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskNdHK51/test-tmp
03:07:25.501 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:07:25.504 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskNdHK51/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:07:25.761 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
03:07:25.907 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:07:25.908 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:07:25.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:25.913383 12065 mini_oidc.cc:150] Starting JWKS server
03:07:25.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:25.917835 12065 webserver.cc:458] Webserver started at http://127.0.0.1:39945/ using document root <none> and password file <none>
03:07:25.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:25.918432 12065 mini_oidc.cc:186] Starting OIDC Discovery server
03:07:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:25.920584 12065 webserver.cc:458] Webserver started at http://127.0.0.1:40363/ using document root <none> and password file <none>
03:07:25.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-14T03:07:25Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
03:07:25.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-14T03:07:25Z Disabled control of system clock
03:07:25.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:25.983853 12065 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:25.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:25.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-0/wal
03:07:25.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-0/data
03:07:25.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:25.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:25.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:25.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:25.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:25.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:25.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-0/logs
03:07:25.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-0/data/info.pb
03:07:25.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:25.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:25.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:25.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:25.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:25.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:25.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:25.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:25.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:25.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:25.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:25.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:25.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.200.126:44413
03:07:25.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.200.126
03:07:25.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:25.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.200.84:38853
03:07:25.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:25.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:25.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:25.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:25.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.200.126:44413,127.11.200.125:39563,127.11.200.124:33463
03:07:25.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:25.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39945/jwks.json with env {}
03:07:26.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:26.283233 12077 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:26.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:26.283833 12077 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:26.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:26.284495 12077 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:26.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:26.315830 12077 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:26.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:26.316126 12077 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:26.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:26.316327 12077 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:26.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:26.316509 12077 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:26.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:26.316745 12077 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:26.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:26.316969 12077 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39945/jwks.json
03:07:26.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:26.352689 12077 master_runner.cc:386] Master server non-default flags:
03:07:26.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:26.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.200.84:38853
03:07:26.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:26.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:26.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-0/data
03:07:26.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-0/wal
03:07:26.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:26.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.200.126:44413,127.11.200.125:39563,127.11.200.124:33463
03:07:26.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:26.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:26.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:26.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.200.126:44413
03:07:26.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:26.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:26.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:26.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39945/jwks.json
03:07:26.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:26.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:26.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-0/data/info.pb
03:07:26.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.200.126
03:07:26.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:26.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:26.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:26.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:26.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:26.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:26.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-0/logs
03:07:26.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:26.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:26.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:26.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:26.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:07:26.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
03:07:26.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:26.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 02:43:23 UTC on a633016ff5c9
03:07:26.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4168
03:07:26.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:26.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:26.354157 12077 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:26.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:26.356268 12077 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:26.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:26.373078 12083 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:27.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:26.373806 12086 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:27.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:26.374186 12084 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:27.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:26.374749 12077 server_base.cc:1034] running on GCE node
03:07:27.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.561194 12077 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:27.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.563910 12077 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:27.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.565239 12077 hybrid_clock.cc:648] HybridClock initialized: now 1736824047565213 us; error 48 us; skew 500 ppm
03:07:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.566046 12077 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:27.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.572080 12077 webserver.cc:458] Webserver started at http://127.11.200.126:43471/ using document root <none> and password file <none>
03:07:27.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.572981 12077 fs_manager.cc:362] Metadata directory not provided
03:07:27.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.573158 12077 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:27.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.573545 12077 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:07:27.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.578346 12077 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-0/data/instance:
03:07:27.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c3bab1647d844f528b526b2aea5dfc5f"
03:07:27.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:27 on dist-test-slave-rqxq"
03:07:27.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.579512 12077 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-0/wal/instance:
03:07:27.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c3bab1647d844f528b526b2aea5dfc5f"
03:07:27.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:27 on dist-test-slave-rqxq"
03:07:27.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.586678 12077 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.000s
03:07:27.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.592365 12093 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:27.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.593430 12077 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
03:07:27.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.593704 12077 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-0/data,/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-0/wal
03:07:27.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c3bab1647d844f528b526b2aea5dfc5f"
03:07:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:27 on dist-test-slave-rqxq"
03:07:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.594002 12077 fs_report.cc:389] FS layout report
03:07:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-0/wal
03:07:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-0/wal
03:07:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-0/data/data
03:07:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:27.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.650625 12077 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:27.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.652078 12077 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:27.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.652499 12077 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:27.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.721406 12077 rpc_server.cc:307] RPC server started. Bound to: 127.11.200.126:44413
03:07:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.721487 12144 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.200.126:44413 every 8 connection(s)
03:07:27.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.724061 12077 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-0/data/info.pb
03:07:27.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.729529 12065 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu as pid 12077
03:07:27.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.729672 12145 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:07:27.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.730098 12065 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-0/wal/instance
03:07:27.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.735365 12065 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:27.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:27.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-1/wal
03:07:27.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-1/data
03:07:27.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:27.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:27.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:27.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:27.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:27.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:27.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-1/logs
03:07:27.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-1/data/info.pb
03:07:27.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:27.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:27.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:27.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:27.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:27.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:27.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:27.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:27.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:27.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:27.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:27.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:27.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.200.125:39563
03:07:27.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.200.125
03:07:27.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:27.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.200.84:38853
03:07:27.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:27.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:27.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:27.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:27.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.200.126:44413,127.11.200.125:39563,127.11.200.124:33463
03:07:27.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:27.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39945/jwks.json with env {}
03:07:27.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.747982 12145 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.200.126" port: 44413 } has no permanent_uuid. Determining permanent_uuid...
03:07:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.775400 12145 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } has no permanent_uuid. Determining permanent_uuid...
03:07:27.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:27.778291 12097 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.200.125:39563: connect: Connection refused (error 111)
03:07:27.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:27.782799 12145 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.200.125:39563: Network error: Client connection negotiation failed: client connection to 127.11.200.125:39563: connect: Connection refused (error 111)
03:07:27.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.832285 12145 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } attempt: 1
03:07:27.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:27.836297 12145 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.200.125:39563: Network error: Client connection negotiation failed: client connection to 127.11.200.125:39563: connect: Connection refused (error 111)
03:07:27.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:27.904690 12145 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } attempt: 2
03:07:27.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:27.908715 12145 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.200.125:39563: Network error: Client connection negotiation failed: client connection to 127.11.200.125:39563: connect: Connection refused (error 111)
03:07:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:28.000135 12145 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } attempt: 3
03:07:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:28.004012 12145 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.200.125:39563: Network error: Client connection negotiation failed: client connection to 127.11.200.125:39563: connect: Connection refused (error 111)
03:07:28.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:28.050995 12147 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:28.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:28.051576 12147 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:28.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:28.052218 12147 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:28.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:28.083292 12147 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:28.083616 12147 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:28.083822 12147 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:28.084007 12147 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:28.084182 12147 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:28.084343 12147 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39945/jwks.json
03:07:28.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:28.118920 12147 master_runner.cc:386] Master server non-default flags:
03:07:28.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:28.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.200.84:38853
03:07:28.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:28.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:28.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-1/data
03:07:28.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-1/wal
03:07:28.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:28.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.200.126:44413,127.11.200.125:39563,127.11.200.124:33463
03:07:28.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:28.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:28.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:28.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.200.125:39563
03:07:28.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:28.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:28.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:28.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39945/jwks.json
03:07:28.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:28.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:28.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-1/data/info.pb
03:07:28.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.200.125
03:07:28.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:28.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:28.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:28.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:28.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:28.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:28.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-1/logs
03:07:28.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:28.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:28.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:28.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:28.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:07:28.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
03:07:28.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:28.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 02:43:23 UTC on a633016ff5c9
03:07:28.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4168
03:07:28.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:28.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:28.120536 12147 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:28.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:28.122614 12147 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:28.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:28.137905 12156 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:28.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:28.147430 12145 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } attempt: 4
03:07:28.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:28.151377 12145 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.200.125:39563: Network error: Client connection negotiation failed: client connection to 127.11.200.125:39563: connect: Connection refused (error 111)
03:07:28.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:28.450861 12145 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } attempt: 5
03:07:28.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:28.456032 12145 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.200.125:39563: Network error: Client connection negotiation failed: client connection to 127.11.200.125:39563: connect: Connection refused (error 111)
03:07:29.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.003518 12145 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } attempt: 6
03:07:29.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:29.021651 12145 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.200.125:39563: Network error: Client connection negotiation failed: client connection to 127.11.200.125:39563: connect: Connection refused (error 111)
03:07:29.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:28.138415 12157 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:29.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:28.139632 12159 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:29.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:28.140185 12147 server_base.cc:1034] running on GCE node
03:07:29.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.333525 12147 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:29.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.336190 12147 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:29.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.337545 12147 hybrid_clock.cc:648] HybridClock initialized: now 1736824049337523 us; error 51 us; skew 500 ppm
03:07:29.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.338539 12147 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:29.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.345176 12147 webserver.cc:458] Webserver started at http://127.11.200.125:34279/ using document root <none> and password file <none>
03:07:29.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.346170 12147 fs_manager.cc:362] Metadata directory not provided
03:07:29.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.346361 12147 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:29.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.346729 12147 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:07:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.351694 12147 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-1/data/instance:
03:07:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "693cb0b186f247e5a892235c6f0a9ee1"
03:07:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:29 on dist-test-slave-rqxq"
03:07:29.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.352862 12147 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-1/wal/instance:
03:07:29.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "693cb0b186f247e5a892235c6f0a9ee1"
03:07:29.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:29 on dist-test-slave-rqxq"
03:07:29.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.360026 12147 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
03:07:29.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.365368 12168 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:29.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.366385 12147 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
03:07:29.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.366717 12147 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-1/data,/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-1/wal
03:07:29.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "693cb0b186f247e5a892235c6f0a9ee1"
03:07:29.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:29 on dist-test-slave-rqxq"
03:07:29.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.367084 12147 fs_report.cc:389] FS layout report
03:07:29.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:29.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-1/wal
03:07:29.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-1/wal
03:07:29.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-1/data/data
03:07:29.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:29.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:29.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.438169 12147 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:29.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.439805 12147 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:29.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.440318 12147 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:29.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.508203 12147 rpc_server.cc:307] RPC server started. Bound to: 127.11.200.125:39563
03:07:29.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.508291 12219 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.200.125:39563 every 8 connection(s)
03:07:29.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.510910 12147 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-1/data/info.pb
03:07:29.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.513183 12065 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu as pid 12147
03:07:29.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.513679 12065 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-1/wal/instance
03:07:29.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.516754 12065 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:29.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:29.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-2/wal
03:07:29.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-2/data
03:07:29.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:29.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:29.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:29.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:29.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:29.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:29.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-2/logs
03:07:29.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-2/data/info.pb
03:07:29.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:29.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:29.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:29.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:29.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:29.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:29.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:29.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:29.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:29.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:29.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:29.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:29.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.200.124:33463
03:07:29.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.200.124
03:07:29.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:29.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.200.84:38853
03:07:29.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:29.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:29.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:29.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:29.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.200.126:44413,127.11.200.125:39563,127.11.200.124:33463
03:07:29.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:29.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39945/jwks.json with env {}
03:07:29.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.517194 12220 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:07:29.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.529531 12220 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.200.126" port: 44413 } has no permanent_uuid. Determining permanent_uuid...
03:07:29.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.552309 12220 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } has no permanent_uuid. Determining permanent_uuid...
03:07:29.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.566941 12220 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } has no permanent_uuid. Determining permanent_uuid...
03:07:29.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:29.569741 12169 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.200.124:33463: connect: Connection refused (error 111)
03:07:29.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:29.581609 12220 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.200.124:33463: Network error: Client connection negotiation failed: client connection to 127.11.200.124:33463: connect: Connection refused (error 111)
03:07:29.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.631203 12220 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } attempt: 1
03:07:29.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:29.635044 12220 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.200.124:33463: Network error: Client connection negotiation failed: client connection to 127.11.200.124:33463: connect: Connection refused (error 111)
03:07:29.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.703528 12220 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } attempt: 2
03:07:29.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:29.707338 12220 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.200.124:33463: Network error: Client connection negotiation failed: client connection to 127.11.200.124:33463: connect: Connection refused (error 111)
03:07:29.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.798867 12220 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } attempt: 3
03:07:29.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:29.803087 12220 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.200.124:33463: Network error: Client connection negotiation failed: client connection to 127.11.200.124:33463: connect: Connection refused (error 111)
03:07:29.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:29.833720 12222 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:29.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:29.834275 12222 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:29.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:29.835014 12222 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:29.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:29.871145 12222 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:29.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:29.871423 12222 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:29.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:29.871668 12222 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:29.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:29.871902 12222 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:29.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:29.872105 12222 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:29.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:29.872254 12222 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39945/jwks.json
03:07:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.908628 12222 master_runner.cc:386] Master server non-default flags:
03:07:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.200.84:38853
03:07:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-2/data
03:07:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-2/wal
03:07:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.200.126:44413,127.11.200.125:39563,127.11.200.124:33463
03:07:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.200.124:33463
03:07:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39945/jwks.json
03:07:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-2/data/info.pb
03:07:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.200.124
03:07:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-2/logs
03:07:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:07:29.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
03:07:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 02:43:23 UTC on a633016ff5c9
03:07:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4168
03:07:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.910102 12222 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:29.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.912277 12222 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:29.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:29.930433 12232 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.946552 12220 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } attempt: 4
03:07:29.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:29.950814 12220 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.200.124:33463: Network error: Client connection negotiation failed: client connection to 127.11.200.124:33463: connect: Connection refused (error 111)
03:07:30.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:30.082321 12145 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } attempt: 7
03:07:30.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:30.094259 12145 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } has no permanent_uuid. Determining permanent_uuid...
03:07:30.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:30.100236 12145 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.200.124:33463: Network error: Client connection negotiation failed: client connection to 127.11.200.124:33463: connect: Connection refused (error 111)
03:07:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:30.158605 12145 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } attempt: 1
03:07:30.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:30.163154 12145 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.200.124:33463: Network error: Client connection negotiation failed: client connection to 127.11.200.124:33463: connect: Connection refused (error 111)
03:07:30.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:30.244557 12145 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } attempt: 2
03:07:30.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:30.249197 12145 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.200.124:33463: Network error: Client connection negotiation failed: client connection to 127.11.200.124:33463: connect: Connection refused (error 111)
03:07:30.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:30.250406 12220 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } attempt: 5
03:07:30.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:30.255307 12220 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.200.124:33463: Network error: Client connection negotiation failed: client connection to 127.11.200.124:33463: connect: Connection refused (error 111)
03:07:30.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:30.334581 12145 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } attempt: 3
03:07:30.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:30.339203 12145 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.200.124:33463: Network error: Client connection negotiation failed: client connection to 127.11.200.124:33463: connect: Connection refused (error 111)
03:07:30.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:30.479638 12145 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } attempt: 4
03:07:30.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:30.484427 12145 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.200.124:33463: Network error: Client connection negotiation failed: client connection to 127.11.200.124:33463: connect: Connection refused (error 111)
03:07:30.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:30.767794 12145 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } attempt: 5
03:07:30.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:30.772328 12145 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.200.124:33463: Network error: Client connection negotiation failed: client connection to 127.11.200.124:33463: connect: Connection refused (error 111)
03:07:30.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:30.802939 12220 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } attempt: 6
03:07:30.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:30.810425 12220 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.200.124:33463: Network error: Client connection negotiation failed: client connection to 127.11.200.124:33463: connect: Connection refused (error 111)
03:07:31.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:29.930900 12233 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:31.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:29.931985 12222 server_base.cc:1034] running on GCE node
03:07:31.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:29.931636 12235 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.139637 12222 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:31.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.142750 12222 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:31.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.144176 12222 hybrid_clock.cc:648] HybridClock initialized: now 1736824051144127 us; error 55 us; skew 500 ppm
03:07:31.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.144930 12222 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:31.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.151516 12222 webserver.cc:458] Webserver started at http://127.11.200.124:38733/ using document root <none> and password file <none>
03:07:31.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.152437 12222 fs_manager.cc:362] Metadata directory not provided
03:07:31.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.152626 12222 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:31.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.153007 12222 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:07:31.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.158023 12222 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-2/data/instance:
03:07:31.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a0b55e029a2d4dfea16959a87847981b"
03:07:31.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:31 on dist-test-slave-rqxq"
03:07:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.159049 12222 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-2/wal/instance:
03:07:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a0b55e029a2d4dfea16959a87847981b"
03:07:31.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:31 on dist-test-slave-rqxq"
03:07:31.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.166430 12222 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.000s
03:07:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.171772 12247 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.172747 12222 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
03:07:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.173075 12222 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-2/data,/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-2/wal
03:07:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a0b55e029a2d4dfea16959a87847981b"
03:07:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:31 on dist-test-slave-rqxq"
03:07:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.173421 12222 fs_report.cc:389] FS layout report
03:07:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-2/wal
03:07:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-2/wal
03:07:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-2/data/data
03:07:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:31.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.231537 12222 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:31.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.233109 12222 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:31.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.233597 12222 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:31.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.302264 12222 rpc_server.cc:307] RPC server started. Bound to: 127.11.200.124:33463
03:07:31.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.302350 12298 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.200.124:33463 every 8 connection(s)
03:07:31.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.305181 12222 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-2/data/info.pb
03:07:31.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.309206 12065 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu as pid 12222
03:07:31.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.309691 12065 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/master-2/wal/instance
03:07:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.311182 12299 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:07:31.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.324687 12145 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } attempt: 6
03:07:31.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.328666 12299 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.200.126" port: 44413 } has no permanent_uuid. Determining permanent_uuid...
03:07:31.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.361604 12299 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } has no permanent_uuid. Determining permanent_uuid...
03:07:31.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.368607 12145 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f: Bootstrap starting.
03:07:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.374001 12299 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } has no permanent_uuid. Determining permanent_uuid...
03:07:31.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.375604 12145 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f: Neither blocks nor log segments found. Creating new log.
03:07:31.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.378320 12145 log.cc:826] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f: Log is configured to *not* fsync() on all Append() calls
03:07:31.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.384016 12145 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f: No bootstrap required, opened a new log
03:07:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.391733 12299 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b: Bootstrap starting.
03:07:31.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.401280 12299 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b: Neither blocks nor log segments found. Creating new log.
03:07:31.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.403821 12299 log.cc:826] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b: Log is configured to *not* fsync() on all Append() calls
03:07:31.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.408277 12299 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b: No bootstrap required, opened a new log
03:07:31.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.410506 12145 raft_consensus.cc:357] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bab1647d844f528b526b2aea5dfc5f" member_type: VOTER last_known_addr { host: "127.11.200.126" port: 44413 } } peers { permanent_uuid: "693cb0b186f247e5a892235c6f0a9ee1" member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } } peers { permanent_uuid: "a0b55e029a2d4dfea16959a87847981b" member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } }
03:07:31.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.411165 12145 raft_consensus.cc:383] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:07:31.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.411440 12145 raft_consensus.cc:738] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c3bab1647d844f528b526b2aea5dfc5f, State: Initialized, Role: FOLLOWER
03:07:31.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.412350 12145 consensus_queue.cc:260] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bab1647d844f528b526b2aea5dfc5f" member_type: VOTER last_known_addr { host: "127.11.200.126" port: 44413 } } peers { permanent_uuid: "693cb0b186f247e5a892235c6f0a9ee1" member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } } peers { permanent_uuid: "a0b55e029a2d4dfea16959a87847981b" member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } }
03:07:31.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.419729 12311 sys_catalog.cc:455] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bab1647d844f528b526b2aea5dfc5f" member_type: VOTER last_known_addr { host: "127.11.200.126" port: 44413 } } peers { permanent_uuid: "693cb0b186f247e5a892235c6f0a9ee1" member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } } peers { permanent_uuid: "a0b55e029a2d4dfea16959a87847981b" member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } } }
03:07:31.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.420919 12311 sys_catalog.cc:458] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f [sys.catalog]: This master's current role is: FOLLOWER
03:07:31.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.421804 12145 sys_catalog.cc:564] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f [sys.catalog]: configured and running, proceeding with master startup.
03:07:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.433588 12299 raft_consensus.cc:357] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bab1647d844f528b526b2aea5dfc5f" member_type: VOTER last_known_addr { host: "127.11.200.126" port: 44413 } } peers { permanent_uuid: "693cb0b186f247e5a892235c6f0a9ee1" member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } } peers { permanent_uuid: "a0b55e029a2d4dfea16959a87847981b" member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } }
03:07:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.434247 12299 raft_consensus.cc:383] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:07:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.434523 12299 raft_consensus.cc:738] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a0b55e029a2d4dfea16959a87847981b, State: Initialized, Role: FOLLOWER
03:07:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.435778 12299 consensus_queue.cc:260] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bab1647d844f528b526b2aea5dfc5f" member_type: VOTER last_known_addr { host: "127.11.200.126" port: 44413 } } peers { permanent_uuid: "693cb0b186f247e5a892235c6f0a9ee1" member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } } peers { permanent_uuid: "a0b55e029a2d4dfea16959a87847981b" member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } }
03:07:31.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.444628 12319 sys_catalog.cc:455] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bab1647d844f528b526b2aea5dfc5f" member_type: VOTER last_known_addr { host: "127.11.200.126" port: 44413 } } peers { permanent_uuid: "693cb0b186f247e5a892235c6f0a9ee1" member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } } peers { permanent_uuid: "a0b55e029a2d4dfea16959a87847981b" member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } } }
03:07:31.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.445993 12319 sys_catalog.cc:458] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [sys.catalog]: This master's current role is: FOLLOWER
03:07:31.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.446846 12299 sys_catalog.cc:564] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [sys.catalog]: configured and running, proceeding with master startup.
03:07:31.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:31.454489 12323 catalog_manager.cc:1559] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:07:31.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:31.458731 12323 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:07:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:31.488519 12253 tablet.cc:2367] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:07:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.451723 12077 master_runner.cc:186] Error getting master registration for 127.11.200.125:39563: OK, instance_id { permanent_uuid: "693cb0b186f247e5a892235c6f0a9ee1" instance_seqno: 1736824049488830 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:07:31.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.493192 12077 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:31.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:31.500481 12335 catalog_manager.cc:1559] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:07:31.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:31.501072 12335 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:07:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.492726 12222 master_runner.cc:186] Error getting master registration for 127.11.200.125:39563: OK, instance_id { permanent_uuid: "693cb0b186f247e5a892235c6f0a9ee1" instance_seqno: 1736824049488830 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:07:31.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.524201 12222 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:31.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.612516 12319 raft_consensus.cc:491] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:07:31.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.613113 12319 raft_consensus.cc:513] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bab1647d844f528b526b2aea5dfc5f" member_type: VOTER last_known_addr { host: "127.11.200.126" port: 44413 } } peers { permanent_uuid: "693cb0b186f247e5a892235c6f0a9ee1" member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } } peers { permanent_uuid: "a0b55e029a2d4dfea16959a87847981b" member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } }
03:07:31.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.616967 12120 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a0b55e029a2d4dfea16959a87847981b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c3bab1647d844f528b526b2aea5dfc5f" is_pre_election: true
03:07:31.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.617017 12319 leader_election.cc:290] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c3bab1647d844f528b526b2aea5dfc5f (127.11.200.126:44413), 693cb0b186f247e5a892235c6f0a9ee1 (127.11.200.125:39563)
03:07:31.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.617980 12120 raft_consensus.cc:2463] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a0b55e029a2d4dfea16959a87847981b in term 0.
03:07:31.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.619318 12248 leader_election.cc:304] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a0b55e029a2d4dfea16959a87847981b, c3bab1647d844f528b526b2aea5dfc5f; no voters:
03:07:31.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.620334 12319 raft_consensus.cc:2798] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [term 0 FOLLOWER]: Leader pre-election won for term 1
03:07:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.621140 12319 raft_consensus.cc:491] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:07:31.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.623494 12319 raft_consensus.cc:3054] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [term 0 FOLLOWER]: Advancing to term 1
03:07:31.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.635661 12319 raft_consensus.cc:513] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bab1647d844f528b526b2aea5dfc5f" member_type: VOTER last_known_addr { host: "127.11.200.126" port: 44413 } } peers { permanent_uuid: "693cb0b186f247e5a892235c6f0a9ee1" member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } } peers { permanent_uuid: "a0b55e029a2d4dfea16959a87847981b" member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } }
03:07:31.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.641577 12319 leader_election.cc:290] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [CANDIDATE]: Term 1 election: Requested vote from peers c3bab1647d844f528b526b2aea5dfc5f (127.11.200.126:44413), 693cb0b186f247e5a892235c6f0a9ee1 (127.11.200.125:39563)
03:07:31.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.644730 12120 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a0b55e029a2d4dfea16959a87847981b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c3bab1647d844f528b526b2aea5dfc5f"
03:07:31.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.645327 12120 raft_consensus.cc:3054] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f [term 0 FOLLOWER]: Advancing to term 1
03:07:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.652390 12120 raft_consensus.cc:2463] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a0b55e029a2d4dfea16959a87847981b in term 1.
03:07:31.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.653350 12248 leader_election.cc:304] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a0b55e029a2d4dfea16959a87847981b, c3bab1647d844f528b526b2aea5dfc5f; no voters:
03:07:31.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.654095 12319 raft_consensus.cc:2798] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [term 1 FOLLOWER]: Leader election won for term 1
03:07:31.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.659883 12319 raft_consensus.cc:695] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [term 1 LEADER]: Becoming Leader. State: Replica: a0b55e029a2d4dfea16959a87847981b, State: Running, Role: LEADER
03:07:31.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:31.661168 12099 tablet.cc:2367] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:07:31.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.661067 12319 consensus_queue.cc:237] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bab1647d844f528b526b2aea5dfc5f" member_type: VOTER last_known_addr { host: "127.11.200.126" port: 44413 } } peers { permanent_uuid: "693cb0b186f247e5a892235c6f0a9ee1" member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } } peers { permanent_uuid: "a0b55e029a2d4dfea16959a87847981b" member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } }
03:07:31.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.677446 12337 sys_catalog.cc:455] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [sys.catalog]: SysCatalogTable state changed. Reason: New leader a0b55e029a2d4dfea16959a87847981b. Latest consensus state: current_term: 1 leader_uuid: "a0b55e029a2d4dfea16959a87847981b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bab1647d844f528b526b2aea5dfc5f" member_type: VOTER last_known_addr { host: "127.11.200.126" port: 44413 } } peers { permanent_uuid: "693cb0b186f247e5a892235c6f0a9ee1" member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } } peers { permanent_uuid: "a0b55e029a2d4dfea16959a87847981b" member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } } }
03:07:31.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.678318 12337 sys_catalog.cc:458] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [sys.catalog]: This master's current role is: LEADER
03:07:31.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.682929 12339 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
03:07:31.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.690261 12339 catalog_manager.cc:1485] Initializing Kudu cluster ID...
03:07:31.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.716426 12120 raft_consensus.cc:1270] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f [term 1 FOLLOWER]: Refusing update from remote peer a0b55e029a2d4dfea16959a87847981b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:07:31.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.718206 12337 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [LEADER]: Connected to new peer: Peer: permanent_uuid: "c3bab1647d844f528b526b2aea5dfc5f" member_type: VOTER last_known_addr { host: "127.11.200.126" port: 44413 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:07:31.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.785826 12340 mvcc.cc:204] Tried to move back new op lower bound from 7114031315810865152 to 7114031315613503488. Current Snapshot: MvccSnapshot[applied={T|T < 7114031315810865152}]
03:07:31.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.788303 12311 sys_catalog.cc:455] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f [sys.catalog]: SysCatalogTable state changed. Reason: New leader a0b55e029a2d4dfea16959a87847981b. Latest consensus state: current_term: 1 leader_uuid: "a0b55e029a2d4dfea16959a87847981b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bab1647d844f528b526b2aea5dfc5f" member_type: VOTER last_known_addr { host: "127.11.200.126" port: 44413 } } peers { permanent_uuid: "693cb0b186f247e5a892235c6f0a9ee1" member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } } peers { permanent_uuid: "a0b55e029a2d4dfea16959a87847981b" member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } } }
03:07:31.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.790462 12311 sys_catalog.cc:458] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f [sys.catalog]: This master's current role is: FOLLOWER
03:07:31.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.791512 12319 sys_catalog.cc:455] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a0b55e029a2d4dfea16959a87847981b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bab1647d844f528b526b2aea5dfc5f" member_type: VOTER last_known_addr { host: "127.11.200.126" port: 44413 } } peers { permanent_uuid: "693cb0b186f247e5a892235c6f0a9ee1" member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } } peers { permanent_uuid: "a0b55e029a2d4dfea16959a87847981b" member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } } }
03:07:31.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.792284 12319 sys_catalog.cc:458] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [sys.catalog]: This master's current role is: LEADER
03:07:31.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.800298 12339 catalog_manager.cc:1348] Generated new cluster ID: 70ad4c43c9fe4db0bd3a5abfe5b3fc96
03:07:31.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.800590 12339 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
03:07:31.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.817065 12311 sys_catalog.cc:455] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a0b55e029a2d4dfea16959a87847981b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bab1647d844f528b526b2aea5dfc5f" member_type: VOTER last_known_addr { host: "127.11.200.126" port: 44413 } } peers { permanent_uuid: "693cb0b186f247e5a892235c6f0a9ee1" member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } } peers { permanent_uuid: "a0b55e029a2d4dfea16959a87847981b" member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } } }
03:07:31.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.817857 12311 sys_catalog.cc:458] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f [sys.catalog]: This master's current role is: FOLLOWER
03:07:31.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.829365 12339 catalog_manager.cc:1371] Generated new certificate authority record
03:07:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.831377 12339 catalog_manager.cc:1505] Loading token signing keys...
03:07:31.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.870988 12220 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } attempt: 7
03:07:31.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.882697 12339 catalog_manager.cc:5899] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b: Generated new TSK 0
03:07:31.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.885929 12339 catalog_manager.cc:1515] Initializing in-progress tserver states...
03:07:31.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.910940 12065 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:31.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:31.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-0/wal
03:07:31.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-0/data
03:07:31.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:31.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:31.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-0/logs
03:07:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-0/data/info.pb
03:07:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:07:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.200.65:0
03:07:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.200.65
03:07:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.200.65
03:07:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.200.126:44413,127.11.200.125:39563,127.11.200.124:33463
03:07:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.200.84:38853
03:07:31.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:31.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:31.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:07:31.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.915825 12220 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 693cb0b186f247e5a892235c6f0a9ee1: Bootstrap starting.
03:07:31.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.923359 12220 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 693cb0b186f247e5a892235c6f0a9ee1: Neither blocks nor log segments found. Creating new log.
03:07:31.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.926088 12220 log.cc:826] T 00000000000000000000000000000000 P 693cb0b186f247e5a892235c6f0a9ee1: Log is configured to *not* fsync() on all Append() calls
03:07:31.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.938429 12220 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 693cb0b186f247e5a892235c6f0a9ee1: No bootstrap required, opened a new log
03:07:31.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.972041 12220 raft_consensus.cc:357] T 00000000000000000000000000000000 P 693cb0b186f247e5a892235c6f0a9ee1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bab1647d844f528b526b2aea5dfc5f" member_type: VOTER last_known_addr { host: "127.11.200.126" port: 44413 } } peers { permanent_uuid: "693cb0b186f247e5a892235c6f0a9ee1" member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } } peers { permanent_uuid: "a0b55e029a2d4dfea16959a87847981b" member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } }
03:07:31.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.973273 12220 raft_consensus.cc:383] T 00000000000000000000000000000000 P 693cb0b186f247e5a892235c6f0a9ee1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:07:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.973774 12220 raft_consensus.cc:738] T 00000000000000000000000000000000 P 693cb0b186f247e5a892235c6f0a9ee1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 693cb0b186f247e5a892235c6f0a9ee1, State: Initialized, Role: FOLLOWER
03:07:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.974905 12220 consensus_queue.cc:260] T 00000000000000000000000000000000 P 693cb0b186f247e5a892235c6f0a9ee1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bab1647d844f528b526b2aea5dfc5f" member_type: VOTER last_known_addr { host: "127.11.200.126" port: 44413 } } peers { permanent_uuid: "693cb0b186f247e5a892235c6f0a9ee1" member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } } peers { permanent_uuid: "a0b55e029a2d4dfea16959a87847981b" member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } }
03:07:31.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.983896 12349 sys_catalog.cc:455] T 00000000000000000000000000000000 P 693cb0b186f247e5a892235c6f0a9ee1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bab1647d844f528b526b2aea5dfc5f" member_type: VOTER last_known_addr { host: "127.11.200.126" port: 44413 } } peers { permanent_uuid: "693cb0b186f247e5a892235c6f0a9ee1" member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } } peers { permanent_uuid: "a0b55e029a2d4dfea16959a87847981b" member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } } }
03:07:31.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.988744 12349 sys_catalog.cc:458] T 00000000000000000000000000000000 P 693cb0b186f247e5a892235c6f0a9ee1 [sys.catalog]: This master's current role is: FOLLOWER
03:07:31.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.988953 12193 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 693cb0b186f247e5a892235c6f0a9ee1 [term 0 FOLLOWER]: Advancing to term 1
03:07:31.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.988394 12220 sys_catalog.cc:564] T 00000000000000000000000000000000 P 693cb0b186f247e5a892235c6f0a9ee1 [sys.catalog]: configured and running, proceeding with master startup.
03:07:31.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.991803 12194 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a0b55e029a2d4dfea16959a87847981b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "693cb0b186f247e5a892235c6f0a9ee1"
03:07:31.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:31.995427 12195 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a0b55e029a2d4dfea16959a87847981b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "693cb0b186f247e5a892235c6f0a9ee1" is_pre_election: true
03:07:32.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:32.001876 12193 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 693cb0b186f247e5a892235c6f0a9ee1 [term 1 FOLLOWER]: Refusing update from remote peer a0b55e029a2d4dfea16959a87847981b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:07:32.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:32.003665 12319 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [LEADER]: Connected to new peer: Peer: permanent_uuid: "693cb0b186f247e5a892235c6f0a9ee1" member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:07:32.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:32.077270 12349 sys_catalog.cc:455] T 00000000000000000000000000000000 P 693cb0b186f247e5a892235c6f0a9ee1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a0b55e029a2d4dfea16959a87847981b. Latest consensus state: current_term: 1 leader_uuid: "a0b55e029a2d4dfea16959a87847981b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bab1647d844f528b526b2aea5dfc5f" member_type: VOTER last_known_addr { host: "127.11.200.126" port: 44413 } } peers { permanent_uuid: "693cb0b186f247e5a892235c6f0a9ee1" member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } } peers { permanent_uuid: "a0b55e029a2d4dfea16959a87847981b" member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } } }
03:07:32.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:32.077451 12360 sys_catalog.cc:455] T 00000000000000000000000000000000 P 693cb0b186f247e5a892235c6f0a9ee1 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a0b55e029a2d4dfea16959a87847981b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bab1647d844f528b526b2aea5dfc5f" member_type: VOTER last_known_addr { host: "127.11.200.126" port: 44413 } } peers { permanent_uuid: "693cb0b186f247e5a892235c6f0a9ee1" member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } } peers { permanent_uuid: "a0b55e029a2d4dfea16959a87847981b" member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } } }
03:07:32.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:32.078089 12349 sys_catalog.cc:458] T 00000000000000000000000000000000 P 693cb0b186f247e5a892235c6f0a9ee1 [sys.catalog]: This master's current role is: FOLLOWER
03:07:32.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:32.078327 12360 sys_catalog.cc:458] T 00000000000000000000000000000000 P 693cb0b186f247e5a892235c6f0a9ee1 [sys.catalog]: This master's current role is: FOLLOWER
03:07:32.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:32.132135 12366 catalog_manager.cc:1260] Loaded cluster ID: 70ad4c43c9fe4db0bd3a5abfe5b3fc96
03:07:32.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:32.132417 12366 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 693cb0b186f247e5a892235c6f0a9ee1: loading cluster ID for follower catalog manager: success
03:07:32.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:32.152128 12366 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 693cb0b186f247e5a892235c6f0a9ee1: acquiring CA information for follower catalog manager: success
03:07:32.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:32.158231 12366 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 693cb0b186f247e5a892235c6f0a9ee1: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:32.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:32.465188 12323 catalog_manager.cc:1260] Loaded cluster ID: 70ad4c43c9fe4db0bd3a5abfe5b3fc96
03:07:32.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:32.465812 12323 catalog_manager.cc:1553] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f: loading cluster ID for follower catalog manager: success
03:07:32.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:32.473678 12323 catalog_manager.cc:1575] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f: acquiring CA information for follower catalog manager: success
03:07:32.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:32.489324 12323 catalog_manager.cc:1603] T 00000000000000000000000000000000 P c3bab1647d844f528b526b2aea5dfc5f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:32.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:32.500793 12319 sys_catalog.cc:455] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a0b55e029a2d4dfea16959a87847981b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3bab1647d844f528b526b2aea5dfc5f" member_type: VOTER last_known_addr { host: "127.11.200.126" port: 44413 } } peers { permanent_uuid: "693cb0b186f247e5a892235c6f0a9ee1" member_type: VOTER last_known_addr { host: "127.11.200.125" port: 39563 } } peers { permanent_uuid: "a0b55e029a2d4dfea16959a87847981b" member_type: VOTER last_known_addr { host: "127.11.200.124" port: 33463 } } }
03:07:32.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:32.502701 12319 sys_catalog.cc:458] T 00000000000000000000000000000000 P a0b55e029a2d4dfea16959a87847981b [sys.catalog]: This master's current role is: LEADER
03:07:32.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:32.528138 12347 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:32.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:32.528781 12347 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:32.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:32.529548 12347 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:32.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:32.587872 12347 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:32.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:32.589238 12347 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.200.65
03:07:32.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:32.652213 12347 tablet_server_runner.cc:78] Tablet server non-default flags:
03:07:32.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.200.84:38853
03:07:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-0/data
03:07:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-0/wal
03:07:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.200.65:0
03:07:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-0/data/info.pb
03:07:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.200.65
03:07:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.200.126:44413,127.11.200.125:39563,127.11.200.124:33463
03:07:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.200.65
03:07:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-0/logs
03:07:32.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:32.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:32.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:32.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:07:32.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:07:32.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
03:07:32.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:32.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 02:43:23 UTC on a633016ff5c9
03:07:32.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4168
03:07:32.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:32.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:32.654083 12347 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:32.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:32.656232 12347 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:32.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:32.696799 12376 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:34.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:34.079957 12375 debug-util.cc:398] Leaking SignalData structure 0x7b0800006bc0 after lost signal to thread 12347
03:07:34.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:34.610930 12375 kernel_stack_watchdog.cc:198] Thread 12347 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:636 for 384ms:
03:07:34.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:07:34.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:07:34.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:07:34.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:07:34.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:32.697945 12377 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:34.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:34.616977 12347 thread.cc:635] GCE (cloud detector) Time spent creating pthread: real 1.923s user 0.491s sys 0.828s
03:07:34.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:34.617419 12347 thread.cc:601] GCE (cloud detector) Time spent starting thread: real 1.924s user 0.491s sys 0.828s
03:07:34.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.627723 12347 server_base.cc:1034] running on GCE node
03:07:34.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:34.628301 12381 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:34.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.629495 12347 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:34.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.631983 12347 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:34.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.633419 12347 hybrid_clock.cc:648] HybridClock initialized: now 1736824054633304 us; error 119 us; skew 500 ppm
03:07:34.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.634260 12347 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.645520 12347 webserver.cc:458] Webserver started at http://127.11.200.65:38587/ using document root <none> and password file <none>
03:07:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.646498 12347 fs_manager.cc:362] Metadata directory not provided
03:07:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.646711 12347 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.647233 12347 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:07:34.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.653414 12347 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-0/data/instance:
03:07:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3de3bf5a0259497186c4d0667dcdb18f"
03:07:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:34 on dist-test-slave-rqxq"
03:07:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.654574 12347 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-0/wal/instance:
03:07:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3de3bf5a0259497186c4d0667dcdb18f"
03:07:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:34 on dist-test-slave-rqxq"
03:07:34.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.662572 12347 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.005s
03:07:34.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.669354 12386 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:34.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.670451 12347 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
03:07:34.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.670784 12347 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-0/data,/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-0/wal
03:07:34.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3de3bf5a0259497186c4d0667dcdb18f"
03:07:34.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:34 on dist-test-slave-rqxq"
03:07:34.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.671118 12347 fs_report.cc:389] FS layout report
03:07:34.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:34.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-0/wal
03:07:34.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-0/wal
03:07:34.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-0/data/data
03:07:34.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:34.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:34.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.728788 12347 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:34.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.730719 12347 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:34.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.731288 12347 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:34.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.744397 12347 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:07:34.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.756662 12347 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:07:34.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.756970 12347 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:07:34.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.757277 12347 ts_tablet_manager.cc:610] Registered 0 tablets
03:07:34.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:34.757503 12347 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:07:35.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:35.012914 12347 rpc_server.cc:307] RPC server started. Bound to: 127.11.200.65:37585
03:07:35.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:35.013046 12501 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.200.65:37585 every 8 connection(s)
03:07:35.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:35.015686 12347 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-0/data/info.pb
03:07:35.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:35.024932 12065 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu as pid 12347
03:07:35.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:35.025425 12065 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-0/wal/instance
03:07:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:35.045020 12065 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-1/wal
03:07:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-1/data
03:07:35.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:35.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:35.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:35.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:35.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:35.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-1/logs
03:07:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-1/data/info.pb
03:07:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:07:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.200.66:0
03:07:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.200.66
03:07:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.200.66
03:07:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.200.126:44413,127.11.200.125:39563,127.11.200.124:33463
03:07:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.200.84:38853
03:07:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:35.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:07:35.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:35.093693 12503 heartbeater.cc:346] Connected to a master server at 127.11.200.125:39563
03:07:35.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:35.094213 12503 heartbeater.cc:463] Registering TS with master...
03:07:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:35.095494 12503 heartbeater.cc:510] Master 127.11.200.125:39563 requested a full tablet report, sending...
03:07:35.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:35.100303 12184 ts_manager.cc:194] Registered new tserver with Master: 3de3bf5a0259497186c4d0667dcdb18f (127.11.200.65:37585)
03:07:35.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:35.101914 12502 heartbeater.cc:346] Connected to a master server at 127.11.200.124:33463
03:07:35.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:35.102260 12502 heartbeater.cc:463] Registering TS with master...
03:07:35.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:35.103092 12502 heartbeater.cc:510] Master 127.11.200.124:33463 requested a full tablet report, sending...
03:07:35.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:35.105491 12264 ts_manager.cc:194] Registered new tserver with Master: 3de3bf5a0259497186c4d0667dcdb18f (127.11.200.65:37585)
03:07:35.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:35.106323 12504 heartbeater.cc:346] Connected to a master server at 127.11.200.126:44413
03:07:35.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:35.106616 12504 heartbeater.cc:463] Registering TS with master...
03:07:35.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:35.108039 12504 heartbeater.cc:510] Master 127.11.200.126:44413 requested a full tablet report, sending...
03:07:35.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:35.108207 12264 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.11.200.65:36095
03:07:35.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:35.110862 12110 ts_manager.cc:194] Registered new tserver with Master: 3de3bf5a0259497186c4d0667dcdb18f (127.11.200.65:37585)
03:07:35.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:35.420459 12510 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:35.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:35.420974 12510 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:35.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:35.421557 12510 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:35.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:35.453699 12510 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:35.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:35.454630 12510 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.200.66
03:07:35.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:35.489050 12510 tablet_server_runner.cc:78] Tablet server non-default flags:
03:07:35.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:35.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.200.84:38853
03:07:35.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:35.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:35.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-1/data
03:07:35.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-1/wal
03:07:35.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:35.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:35.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.200.66:0
03:07:35.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:35.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:35.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:35.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-1/data/info.pb
03:07:35.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.200.66
03:07:35.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:35.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.200.126:44413,127.11.200.125:39563,127.11.200.124:33463
03:07:35.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:35.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:35.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:35.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.200.66
03:07:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-1/logs
03:07:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:35.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:07:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:07:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
03:07:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 02:43:23 UTC on a633016ff5c9
03:07:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4168
03:07:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:35.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:35.490728 12510 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:35.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:35.492960 12510 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:35.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:35.514844 12519 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:36.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:36.112726 12502 heartbeater.cc:502] Master 127.11.200.124:33463 was elected leader, sending a full tablet report...
03:07:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:35.515599 12518 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:36.855937 12520 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1341 milliseconds
03:07:36.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:36.858237 12521 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:36.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:36.859997 12510 thread.cc:635] OpenStack (cloud detector) Time spent creating pthread: real 1.348s user 0.001s sys 0.007s
03:07:36.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:36.860356 12510 thread.cc:601] OpenStack (cloud detector) Time spent starting thread: real 1.349s user 0.001s sys 0.007s
03:07:36.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:36.860785 12510 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:36.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:36.864811 12510 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:36.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:36.867830 12510 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:36.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:36.869374 12510 hybrid_clock.cc:648] HybridClock initialized: now 1736824056869325 us; error 38 us; skew 500 ppm
03:07:36.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:36.870610 12510 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:36.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:36.879887 12510 webserver.cc:458] Webserver started at http://127.11.200.66:42011/ using document root <none> and password file <none>
03:07:36.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:36.881186 12510 fs_manager.cc:362] Metadata directory not provided
03:07:36.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:36.881470 12510 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:36.882056 12510 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:07:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:36.889925 12510 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-1/data/instance:
03:07:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f5fb75027ee4ecf8cad4a27ae72296e"
03:07:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:36 on dist-test-slave-rqxq"
03:07:36.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:36.891695 12510 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-1/wal/instance:
03:07:36.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f5fb75027ee4ecf8cad4a27ae72296e"
03:07:36.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:36 on dist-test-slave-rqxq"
03:07:36.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:36.901839 12510 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.001s sys 0.008s
03:07:36.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:36.909960 12530 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:36.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:36.911213 12510 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
03:07:36.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:36.911654 12510 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-1/data,/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-1/wal
03:07:36.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7f5fb75027ee4ecf8cad4a27ae72296e"
03:07:36.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:36 on dist-test-slave-rqxq"
03:07:36.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:36.912122 12510 fs_report.cc:389] FS layout report
03:07:36.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:36.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-1/wal
03:07:36.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-1/wal
03:07:36.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-1/data/data
03:07:36.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:36.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:36.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:36.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:36.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:36.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:36.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:36.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:36.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:36.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:36.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:37.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.015931 12510 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:37.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.017379 12510 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:37.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.017761 12510 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:37.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.020576 12510 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:07:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.024611 12510 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:07:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.024794 12510 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:07:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.024986 12510 ts_tablet_manager.cc:610] Registered 0 tablets
03:07:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.025120 12510 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:07:37.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.174645 12510 rpc_server.cc:307] RPC server started. Bound to: 127.11.200.66:36685
03:07:37.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.174749 12642 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.200.66:36685 every 8 connection(s)
03:07:37.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.177326 12510 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-1/data/info.pb
03:07:37.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.183182 12065 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu as pid 12510
03:07:37.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.183866 12065 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-1/wal/instance
03:07:37.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.220155 12065 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:37.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:37.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-2/wal
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-2/data
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-2/logs
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-2/data/info.pb
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.200.67:0
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.200.67
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.200.67
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.200.126:44413,127.11.200.125:39563,127.11.200.124:33463
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.200.84:38853
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:07:37.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.223716 12644 heartbeater.cc:346] Connected to a master server at 127.11.200.125:39563
03:07:37.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.224217 12644 heartbeater.cc:463] Registering TS with master...
03:07:37.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.225677 12644 heartbeater.cc:510] Master 127.11.200.125:39563 requested a full tablet report, sending...
03:07:37.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.226588 12643 heartbeater.cc:346] Connected to a master server at 127.11.200.124:33463
03:07:37.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.226913 12643 heartbeater.cc:463] Registering TS with master...
03:07:37.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.227778 12643 heartbeater.cc:510] Master 127.11.200.124:33463 requested a full tablet report, sending...
03:07:37.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.228793 12184 ts_manager.cc:194] Registered new tserver with Master: 7f5fb75027ee4ecf8cad4a27ae72296e (127.11.200.66:36685)
03:07:37.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.232208 12264 ts_manager.cc:194] Registered new tserver with Master: 7f5fb75027ee4ecf8cad4a27ae72296e (127.11.200.66:36685)
03:07:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.233753 12264 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.11.200.66:42881
03:07:37.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.236835 12645 heartbeater.cc:346] Connected to a master server at 127.11.200.126:44413
03:07:37.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.237131 12645 heartbeater.cc:463] Registering TS with master...
03:07:37.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.237666 12645 heartbeater.cc:510] Master 127.11.200.126:44413 requested a full tablet report, sending...
03:07:37.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.239661 12110 ts_manager.cc:194] Registered new tserver with Master: 7f5fb75027ee4ecf8cad4a27ae72296e (127.11.200.66:36685)
03:07:37.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:37.540908 12653 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:37.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:37.541442 12653 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:37.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:37.542018 12653 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:37.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:37.572738 12653 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:37.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:37.573661 12653 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.200.67
03:07:37.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.608903 12653 tablet_server_runner.cc:78] Tablet server non-default flags:
03:07:37.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:37.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.200.84:38853
03:07:37.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:37.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:37.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-2/data
03:07:37.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-2/wal
03:07:37.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:37.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:37.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.200.67:0
03:07:37.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:37.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:37.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:37.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-2/data/info.pb
03:07:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.200.67
03:07:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.200.126:44413,127.11.200.125:39563,127.11.200.124:33463
03:07:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.200.67
03:07:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-2/logs
03:07:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:07:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:07:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
03:07:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 02:43:23 UTC on a633016ff5c9
03:07:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4168
03:07:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.610577 12653 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:37.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.612244 12653 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:37.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:37.630453 12660 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:38.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.237874 12643 heartbeater.cc:502] Master 127.11.200.124:33463 was elected leader, sending a full tablet report...
03:07:38.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:37.630441 12659 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:38.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:37.633549 12653 server_base.cc:1034] running on GCE node
03:07:38.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:37.632722 12662 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:38.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.809721 12653 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:38.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.812419 12653 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:38.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.813835 12653 hybrid_clock.cc:648] HybridClock initialized: now 1736824058813799 us; error 56 us; skew 500 ppm
03:07:38.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.814606 12653 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:38.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.820640 12653 webserver.cc:458] Webserver started at http://127.11.200.67:44155/ using document root <none> and password file <none>
03:07:38.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.821487 12653 fs_manager.cc:362] Metadata directory not provided
03:07:38.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.821681 12653 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:38.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.822044 12653 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:07:38.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.826838 12653 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-2/data/instance:
03:07:38.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7fea3b6914b8482c9657853f13781587"
03:07:38.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:38 on dist-test-slave-rqxq"
03:07:38.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.827934 12653 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-2/wal/instance:
03:07:38.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7fea3b6914b8482c9657853f13781587"
03:07:38.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:38 on dist-test-slave-rqxq"
03:07:38.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.834915 12653 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
03:07:38.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.840585 12670 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:38.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.841593 12653 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.005s
03:07:38.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.841878 12653 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-2/data,/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-2/wal
03:07:38.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7fea3b6914b8482c9657853f13781587"
03:07:38.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:38 on dist-test-slave-rqxq"
03:07:38.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.842175 12653 fs_report.cc:389] FS layout report
03:07:38.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:38.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-2/wal
03:07:38.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-2/wal
03:07:38.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-2/data/data
03:07:38.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:38.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:38.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:38.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:38.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:38.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:38.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:38.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:38.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:38.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:38.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:38.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.894461 12653 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:38.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.895907 12653 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:38.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.896299 12653 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:38.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.898703 12653 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:07:38.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.902705 12653 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:07:38.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.902894 12653 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:07:38.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.903084 12653 ts_tablet_manager.cc:610] Registered 0 tablets
03:07:38.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:38.903224 12653 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:07:39.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:39.056305 12653 rpc_server.cc:307] RPC server started. Bound to: 127.11.200.67:37183
03:07:39.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:39.056960 12782 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.200.67:37183 every 8 connection(s)
03:07:39.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:39.058666 12653 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-2/data/info.pb
03:07:39.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:39.067739 12065 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu as pid 12653
03:07:39.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:39.068208 12065 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster7287344663497028042/ts-2/wal/instance
03:07:39.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:39.096539 12783 heartbeater.cc:346] Connected to a master server at 127.11.200.124:33463
03:07:39.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:39.097043 12783 heartbeater.cc:463] Registering TS with master...
03:07:39.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:39.098304 12783 heartbeater.cc:510] Master 127.11.200.124:33463 requested a full tablet report, sending...
03:07:39.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:39.099593 12785 heartbeater.cc:346] Connected to a master server at 127.11.200.126:44413
03:07:39.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:39.099916 12785 heartbeater.cc:463] Registering TS with master...
03:07:39.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:39.100844 12264 ts_manager.cc:194] Registered new tserver with Master: 7fea3b6914b8482c9657853f13781587 (127.11.200.67:37183)
03:07:39.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:39.101034 12784 heartbeater.cc:346] Connected to a master server at 127.11.200.125:39563
03:07:39.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:39.101205 12785 heartbeater.cc:510] Master 127.11.200.126:44413 requested a full tablet report, sending...
03:07:39.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:39.101357 12784 heartbeater.cc:463] Registering TS with master...
03:07:39.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:39.102315 12784 heartbeater.cc:510] Master 127.11.200.125:39563 requested a full tablet report, sending...
03:07:39.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:39.102522 12264 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.11.200.67:38581
03:07:39.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:39.103683 12110 ts_manager.cc:194] Registered new tserver with Master: 7fea3b6914b8482c9657853f13781587 (127.11.200.67:37183)
03:07:39.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:39.104935 12184 ts_manager.cc:194] Registered new tserver with Master: 7fea3b6914b8482c9657853f13781587 (127.11.200.67:37183)
03:07:39.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:39.114446 12065 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
03:07:39.126 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:07:39.127 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
03:07:39.137 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.11.200.126"
port: 44413
}
bound_http_address {
host: "127.11.200.126"
port: 43471
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.11.200.125"
port: 39563
}
bound_http_address {
host: "127.11.200.125"
port: 34279
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.11.200.124"
port: 33463
}
bound_http_address {
host: "127.11.200.124"
port: 38733
}
}
}
03:07:39.141 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:07:39.143 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.11.200.65"
port: 37585
}
bound_http_address {
host: "127.11.200.65"
port: 38587
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.11.200.66"
port: 36685
}
bound_http_address {
host: "127.11.200.66"
port: 42011
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.11.200.67"
port: 37183
}
bound_http_address {
host: "127.11.200.67"
port: 44155
}
}
}
03:07:39.143 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
03:07:39.408 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
03:07:39.478 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskNdHK51/test-tmp
03:07:39.478 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:07:39.478 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskNdHK51/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:07:39.482 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
03:07:39.837 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:39.838 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:07:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:39.841352 12794 mini_oidc.cc:150] Starting JWKS server
03:07:39.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:39.844739 12794 webserver.cc:458] Webserver started at http://127.0.0.1:34021/ using document root <none> and password file <none>
03:07:39.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:39.845149 12794 mini_oidc.cc:186] Starting OIDC Discovery server
03:07:39.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:39.846783 12794 webserver.cc:458] Webserver started at http://127.0.0.1:33417/ using document root <none> and password file <none>
03:07:39.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
03:07:39.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/principal' for realm 'KRBTEST.COM',
03:07:39.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
03:07:39.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:39 dist-test-slave-rqxq krb5kdc[12807](info): setting up network...
03:07:39.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
03:07:39.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:39 dist-test-slave-rqxq krb5kdc[12807](info): set up 2 sockets
03:07:39.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:39 dist-test-slave-rqxq krb5kdc[12807](info): commencing operation
03:07:39.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
03:07:40.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:40.106050 12783 heartbeater.cc:502] Master 127.11.200.124:33463 was elected leader, sending a full tablet report...
03:07:40.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:40.949836 12794 mini_kdc.cc:121] Time spent starting KDC: real 1.064s user 0.002s sys 0.015s
03:07:40.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
03:07:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:07:40.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
03:07:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
03:07:41.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:07:41.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
03:07:41.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
03:07:41.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:07:41.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
03:07:41.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:07:41.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/test-user.keytab.
03:07:41.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/test-user.keytab.
03:07:41.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:41 dist-test-slave-rqxq krb5kdc[12807](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824061, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:41.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:07:41.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-14T03:07:41Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
03:07:41.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-14T03:07:41Z Disabled control of system clock
03:07:41.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.126.190@KRBTEST.COM; defaulting to no policy
03:07:41.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:41.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.126.190@KRBTEST.COM" created.
03:07:41.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:41.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.126.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.190.keytab.
03:07:41.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.126.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.190.keytab.
03:07:41.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:41.134300 12794 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:41.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:41.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/wal
03:07:41.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/data
03:07:41.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:41.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:41.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:41.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:41.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:41.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:41.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/logs
03:07:41.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/data/info.pb
03:07:41.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.126.190:35933
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.126.190
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.190.keytab
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.126.190
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.126.148:44839
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34021/jwks.json
03:07:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:41.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:41.442978 12820 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:41.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:41.443529 12820 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:41.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:41.444501 12820 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:41.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:41.476166 12820 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:41.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:41.476400 12820 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:41.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:41.476723 12820 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:41.477033 12820 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:41.477290 12820 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:41.477484 12820 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:41.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:41.477630 12820 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34021/jwks.json
03:07:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:41.515041 12820 master_runner.cc:386] Master server non-default flags:
03:07:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.126.148:44839
03:07:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/data
03:07:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/wal
03:07:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957
03:07:41.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.190.keytab
03:07:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.126.190
03:07:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.126.190:35933
03:07:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34021/jwks.json
03:07:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/data/info.pb
03:07:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.126.190
03:07:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:41.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/logs
03:07:41.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:41.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:41.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:41.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:41.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:07:41.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
03:07:41.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:41.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 02:43:23 UTC on a633016ff5c9
03:07:41.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4168
03:07:41.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:41.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:41.516306 12820 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:41.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:41.517333 12820 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:41.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:41.536788 12826 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:42.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:41.537281 12827 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:42.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:41.540387 12829 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:42.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:42.844858 12828 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1304 milliseconds
03:07:42.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:42.844965 12820 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:42.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:42.846088 12820 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:42.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:42.849186 12820 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:42.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:42.850606 12820 hybrid_clock.cc:648] HybridClock initialized: now 1736824062850567 us; error 56 us; skew 500 ppm
03:07:42.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:42.851372 12820 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:42.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:42 dist-test-slave-rqxq krb5kdc[12807](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824062, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:42.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:42.857568 12820 init.cc:377] Logged in from keytab as kudu/127.12.126.190@KRBTEST.COM (short username kudu)
03:07:42.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:42.863242 12820 webserver.cc:458] Webserver started at http://127.12.126.190:42053/ using document root <none> and password file <none>
03:07:42.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:42.864143 12820 fs_manager.cc:362] Metadata directory not provided
03:07:42.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:42.864336 12820 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:42.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:42.864722 12820 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:07:42.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:42.869388 12820 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/data/instance:
03:07:42.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4"
03:07:42.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:42 on dist-test-slave-rqxq"
03:07:42.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:42.870438 12820 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/wal/instance:
03:07:42.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4"
03:07:42.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:42 on dist-test-slave-rqxq"
03:07:42.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:42.877925 12820 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
03:07:42.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:42.883018 12838 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:42.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:42.883997 12820 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
03:07:42.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:42.884291 12820 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/data,/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/wal
03:07:42.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4"
03:07:42.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:42 on dist-test-slave-rqxq"
03:07:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:42.884580 12820 fs_report.cc:389] FS layout report
03:07:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/wal
03:07:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/wal
03:07:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/data/data
03:07:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:42.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:42.931036 12820 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:42.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:42.936514 12820 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:42.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:42.937355 12820 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:43.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:43.011631 12820 rpc_server.cc:307] RPC server started. Bound to: 127.12.126.190:35933
03:07:43.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:43.011731 12889 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.126.190:35933 every 8 connection(s)
03:07:43.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:43.015672 12820 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/data/info.pb
03:07:43.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:43.022524 12890 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:07:43.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:43.024327 12794 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu as pid 12820
03:07:43.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:43.024945 12794 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/wal/instance
03:07:43.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:43.038688 12890 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } has no permanent_uuid. Determining permanent_uuid...
03:07:43.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.126.189@KRBTEST.COM; defaulting to no policy
03:07:43.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:43.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.126.189@KRBTEST.COM" created.
03:07:43.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:43 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824062, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.190@KRBTEST.COM for kudu/127.12.126.190@KRBTEST.COM
03:07:43.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:43.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.126.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.189.keytab.
03:07:43.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.126.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.189.keytab.
03:07:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:43.074375 12794 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/wal
03:07:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/data
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/logs
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/data/info.pb
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.126.189:35635
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.126.189
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.189.keytab
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.126.189
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:43.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:43.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.126.148:44839
03:07:43.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:43.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:43.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:43.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:43.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957
03:07:43.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:43.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34021/jwks.json
03:07:43.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:43.078177 12890 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } has no permanent_uuid. Determining permanent_uuid...
03:07:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:43.080379 12841 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.126.189:35635: connect: Connection refused (error 111)
03:07:43.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:43.085467 12890 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.126.189:35635: Network error: Client connection negotiation failed: client connection to 127.12.126.189:35635: connect: Connection refused (error 111)
03:07:43.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:43.134912 12890 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } attempt: 1
03:07:43.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:43.138641 12890 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.126.189:35635: Network error: Client connection negotiation failed: client connection to 127.12.126.189:35635: connect: Connection refused (error 111)
03:07:43.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:43.207079 12890 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } attempt: 2
03:07:43.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:43.212123 12890 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.126.189:35635: Network error: Client connection negotiation failed: client connection to 127.12.126.189:35635: connect: Connection refused (error 111)
03:07:43.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:43.303532 12890 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } attempt: 3
03:07:43.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:43.307754 12890 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.126.189:35635: Network error: Client connection negotiation failed: client connection to 127.12.126.189:35635: connect: Connection refused (error 111)
03:07:43.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:43.394976 12896 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:43.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:43.395511 12896 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:43.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:43.396136 12896 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:43.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:43.426834 12896 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:43.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:43.427140 12896 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:43.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:43.427490 12896 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:43.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:43.428025 12896 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:43.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:43.428382 12896 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:43.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:43.428719 12896 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:43.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:43.428957 12896 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34021/jwks.json
03:07:43.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:43.451172 12890 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } attempt: 4
03:07:43.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:43.454985 12890 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.126.189:35635: Network error: Client connection negotiation failed: client connection to 127.12.126.189:35635: connect: Connection refused (error 111)
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:43.465655 12896 master_runner.cc:386] Master server non-default flags:
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.126.148:44839
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/data
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/wal
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.189.keytab
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.126.189
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.126.189:35635
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34021/jwks.json
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/data/info.pb
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.126.189
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/logs
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:07:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
03:07:43.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:43.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 02:43:23 UTC on a633016ff5c9
03:07:43.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4168
03:07:43.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:43.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:43.466904 12896 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:43.467929 12896 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:43.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:43.484344 12903 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:43.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:43.754444 12890 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } attempt: 5
03:07:43.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:43.759568 12890 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.126.189:35635: Network error: Client connection negotiation failed: client connection to 127.12.126.189:35635: connect: Connection refused (error 111)
03:07:44.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.306967 12890 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } attempt: 6
03:07:44.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:44.319756 12890 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.126.189:35635: Network error: Client connection negotiation failed: client connection to 127.12.126.189:35635: connect: Connection refused (error 111)
03:07:44.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:43.484736 12904 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:44.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:43.485997 12896 server_base.cc:1034] running on GCE node
03:07:44.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:43.485329 12906 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:44.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.699936 12896 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:44.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.702452 12896 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:44.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.703780 12896 hybrid_clock.cc:648] HybridClock initialized: now 1736824064703759 us; error 45 us; skew 500 ppm
03:07:44.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.704628 12896 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:44.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:44 dist-test-slave-rqxq krb5kdc[12807](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824064, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:44.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.710958 12896 init.cc:377] Logged in from keytab as kudu/127.12.126.189@KRBTEST.COM (short username kudu)
03:07:44.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.715907 12896 webserver.cc:458] Webserver started at http://127.12.126.189:38527/ using document root <none> and password file <none>
03:07:44.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.716769 12896 fs_manager.cc:362] Metadata directory not provided
03:07:44.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.716946 12896 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:44.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.717339 12896 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:07:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.721990 12896 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/data/instance:
03:07:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea2f415b454a4cee864155fdd4bbb343"
03:07:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:44 on dist-test-slave-rqxq"
03:07:44.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.723086 12896 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/wal/instance:
03:07:44.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea2f415b454a4cee864155fdd4bbb343"
03:07:44.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:44 on dist-test-slave-rqxq"
03:07:44.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.731719 12896 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
03:07:44.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.738169 12916 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:44.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.739275 12896 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
03:07:44.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.739602 12896 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/data,/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/wal
03:07:44.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea2f415b454a4cee864155fdd4bbb343"
03:07:44.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:44 on dist-test-slave-rqxq"
03:07:44.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.739913 12896 fs_report.cc:389] FS layout report
03:07:44.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:44.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/wal
03:07:44.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/wal
03:07:44.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/data/data
03:07:44.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:44.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:44.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:44.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:44.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:44.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:44.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:44.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:44.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:44.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:44.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:44.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.802942 12896 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:44.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.808372 12896 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:44.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.808820 12896 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:44.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.887176 12896 rpc_server.cc:307] RPC server started. Bound to: 127.12.126.189:35635
03:07:44.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.887252 12967 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.126.189:35635 every 8 connection(s)
03:07:44.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.889796 12896 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/data/info.pb
03:07:44.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.895859 12968 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:07:44.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.899056 12794 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu as pid 12896
03:07:44.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.899456 12794 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/wal/instance
03:07:44.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.907979 12968 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } has no permanent_uuid. Determining permanent_uuid...
03:07:44.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.126.188@KRBTEST.COM; defaulting to no policy
03:07:44.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:44.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.126.188@KRBTEST.COM" created.
03:07:44.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:44 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824064, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.189@KRBTEST.COM for kudu/127.12.126.190@KRBTEST.COM
03:07:44.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:44.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.126.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.188.keytab.
03:07:44.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.126.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.188.keytab.
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.944048 12794 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/wal
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/data
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/logs
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/data/info.pb
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.126.188:40957
03:07:44.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.126.188
03:07:44.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:44.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.188.keytab
03:07:44.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.126.188
03:07:44.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:44.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:44.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:44.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.126.148:44839
03:07:44.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:44.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:44.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:44.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:44.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957
03:07:44.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:44.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34021/jwks.json
03:07:44.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:44.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.949662 12968 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } has no permanent_uuid. Determining permanent_uuid...
03:07:44.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:44 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824064, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.189@KRBTEST.COM for kudu/127.12.126.189@KRBTEST.COM
03:07:44.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:44.976153 12968 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } has no permanent_uuid. Determining permanent_uuid...
03:07:44.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:44.978646 12920 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.126.188:40957: connect: Connection refused (error 111)
03:07:44.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:44.983381 12968 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.126.188:40957: Network error: Client connection negotiation failed: client connection to 127.12.126.188:40957: connect: Connection refused (error 111)
03:07:45.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:45.032886 12968 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } attempt: 1
03:07:45.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:45.038151 12968 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.126.188:40957: Network error: Client connection negotiation failed: client connection to 127.12.126.188:40957: connect: Connection refused (error 111)
03:07:45.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:45.106568 12968 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } attempt: 2
03:07:45.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:45.110183 12968 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.126.188:40957: Network error: Client connection negotiation failed: client connection to 127.12.126.188:40957: connect: Connection refused (error 111)
03:07:45.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:45.201619 12968 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } attempt: 3
03:07:45.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:45.205637 12968 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.126.188:40957: Network error: Client connection negotiation failed: client connection to 127.12.126.188:40957: connect: Connection refused (error 111)
03:07:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:45.292979 12974 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:45.293639 12974 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:45.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:45.294329 12974 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:45.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:45.326004 12974 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:45.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:45.326272 12974 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:45.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:45.326697 12974 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:45.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:45.327003 12974 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:45.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:45.327322 12974 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:45.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:45.327628 12974 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:45.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:45.327845 12974 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34021/jwks.json
03:07:45.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:45.349064 12968 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } attempt: 4
03:07:45.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:45.353055 12968 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.126.188:40957: Network error: Client connection negotiation failed: client connection to 127.12.126.188:40957: connect: Connection refused (error 111)
03:07:45.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:45.365252 12974 master_runner.cc:386] Master server non-default flags:
03:07:45.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:45.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.126.148:44839
03:07:45.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:45.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:45.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/data
03:07:45.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/wal
03:07:45.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957
03:07:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.188.keytab
03:07:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.126.188
03:07:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.126.188:40957
03:07:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34021/jwks.json
03:07:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/data/info.pb
03:07:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.126.188
03:07:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/logs
03:07:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:07:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
03:07:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 02:43:23 UTC on a633016ff5c9
03:07:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4168
03:07:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:45.366616 12974 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:45.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:45.367620 12974 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:45.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:45.380215 12890 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } attempt: 7
03:07:45.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:45.386090 12983 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:45.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:45 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824062, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.190@KRBTEST.COM for kudu/127.12.126.189@KRBTEST.COM
03:07:45.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:45.405261 12890 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } has no permanent_uuid. Determining permanent_uuid...
03:07:45.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:45.411731 12890 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.126.188:40957: Network error: Client connection negotiation failed: client connection to 127.12.126.188:40957: connect: Connection refused (error 111)
03:07:45.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:45.470103 12890 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } attempt: 1
03:07:45.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:45.474475 12890 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.126.188:40957: Network error: Client connection negotiation failed: client connection to 127.12.126.188:40957: connect: Connection refused (error 111)
03:07:45.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:45.555903 12890 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } attempt: 2
03:07:45.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:45.561429 12890 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.126.188:40957: Network error: Client connection negotiation failed: client connection to 127.12.126.188:40957: connect: Connection refused (error 111)
03:07:45.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:45.646803 12890 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } attempt: 3
03:07:45.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:45.651692 12890 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.126.188:40957: Network error: Client connection negotiation failed: client connection to 127.12.126.188:40957: connect: Connection refused (error 111)
03:07:45.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:45.652431 12968 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } attempt: 5
03:07:45.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:45.656939 12968 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.126.188:40957: Network error: Client connection negotiation failed: client connection to 127.12.126.188:40957: connect: Connection refused (error 111)
03:07:45.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:45.792057 12890 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } attempt: 4
03:07:45.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:45.796954 12890 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.126.188:40957: Network error: Client connection negotiation failed: client connection to 127.12.126.188:40957: connect: Connection refused (error 111)
03:07:46.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.080322 12890 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } attempt: 5
03:07:46.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:46.084633 12890 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.126.188:40957: Network error: Client connection negotiation failed: client connection to 127.12.126.188:40957: connect: Connection refused (error 111)
03:07:46.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.204339 12968 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } attempt: 6
03:07:46.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:46.212230 12968 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.126.188:40957: Network error: Client connection negotiation failed: client connection to 127.12.126.188:40957: connect: Connection refused (error 111)
03:07:46.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:45.386515 12982 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:46.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:45.388149 12985 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:46.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:45.387693 12974 server_base.cc:1034] running on GCE node
03:07:46.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.634527 12974 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:46.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.636989 12974 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:46.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.637208 12890 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } attempt: 6
03:07:46.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.638375 12974 hybrid_clock.cc:648] HybridClock initialized: now 1736824066638328 us; error 66 us; skew 500 ppm
03:07:46.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.639626 12974 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:46.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:46.644049 12890 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.126.188:40957: Network error: Client connection negotiation failed: client connection to 127.12.126.188:40957: connect: Connection refused (error 111)
03:07:46.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:46 dist-test-slave-rqxq krb5kdc[12807](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824066, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:46.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.646222 12974 init.cc:377] Logged in from keytab as kudu/127.12.126.188@KRBTEST.COM (short username kudu)
03:07:46.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.651981 12974 webserver.cc:458] Webserver started at http://127.12.126.188:35901/ using document root <none> and password file <none>
03:07:46.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.652848 12974 fs_manager.cc:362] Metadata directory not provided
03:07:46.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.653031 12974 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:46.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.653398 12974 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:07:46.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.658069 12974 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/data/instance:
03:07:46.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "16e48fc0bf4746988b005121cc02d88c"
03:07:46.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:46 on dist-test-slave-rqxq"
03:07:46.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.659049 12974 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/wal/instance:
03:07:46.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "16e48fc0bf4746988b005121cc02d88c"
03:07:46.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:46 on dist-test-slave-rqxq"
03:07:46.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.666299 12974 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
03:07:46.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.671929 13002 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:46.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.672938 12974 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
03:07:46.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.673238 12974 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/data,/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/wal
03:07:46.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "16e48fc0bf4746988b005121cc02d88c"
03:07:46.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:46 on dist-test-slave-rqxq"
03:07:46.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.673576 12974 fs_report.cc:389] FS layout report
03:07:46.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/wal
03:07:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/wal
03:07:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/data/data
03:07:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:46.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:46.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.732903 12974 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:46.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.738431 12974 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:46.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.738824 12974 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:46.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.815616 12974 rpc_server.cc:307] RPC server started. Bound to: 127.12.126.188:40957
03:07:46.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.815747 13053 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.126.188:40957 every 8 connection(s)
03:07:46.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.818215 12974 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/data/info.pb
03:07:46.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.822626 12794 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu as pid 12974
03:07:46.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.823079 12794 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/wal/instance
03:07:46.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.826475 13054 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:07:46.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.847429 13054 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } has no permanent_uuid. Determining permanent_uuid...
03:07:46.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:46 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824061, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.12.126.190@KRBTEST.COM
03:07:46.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:46 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824066, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.188@KRBTEST.COM for kudu/127.12.126.190@KRBTEST.COM
03:07:46.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:46 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824061, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.12.126.189@KRBTEST.COM
03:07:46.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.883428 13054 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } has no permanent_uuid. Determining permanent_uuid...
03:07:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:46 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824066, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.188@KRBTEST.COM for kudu/127.12.126.189@KRBTEST.COM
03:07:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.907934 13054 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } has no permanent_uuid. Determining permanent_uuid...
03:07:46.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:46 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824061, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.12.126.188@KRBTEST.COM
03:07:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:46 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824066, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.188@KRBTEST.COM for kudu/127.12.126.188@KRBTEST.COM
03:07:46.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.938302 13054 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c: Bootstrap starting.
03:07:46.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.945169 13054 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c: Neither blocks nor log segments found. Creating new log.
03:07:46.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.947070 13054 log.cc:826] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c: Log is configured to *not* fsync() on all Append() calls
03:07:46.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.951825 13054 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c: No bootstrap required, opened a new log
03:07:46.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.973923 13054 raft_consensus.cc:357] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } }
03:07:46.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.974581 13054 raft_consensus.cc:383] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:07:46.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.974885 13054 raft_consensus.cc:738] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 16e48fc0bf4746988b005121cc02d88c, State: Initialized, Role: FOLLOWER
03:07:46.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.975802 13054 consensus_queue.cc:260] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } }
03:07:46.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.979064 13065 sys_catalog.cc:455] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } } }
03:07:46.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.979682 13065 sys_catalog.cc:458] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [sys.catalog]: This master's current role is: FOLLOWER
03:07:46.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:46.980543 13054 sys_catalog.cc:564] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [sys.catalog]: configured and running, proceeding with master startup.
03:07:46.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:46.994884 13008 tablet.cc:2367] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:07:47.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.003443 13065 raft_consensus.cc:491] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:07:47.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.003955 13065 raft_consensus.cc:513] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } }
03:07:47.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.006886 13065 leader_election.cc:290] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b32c0d9df04f4be0b9d7658cf31ee9d4 (127.12.126.190:35933), ea2f415b454a4cee864155fdd4bbb343 (127.12.126.189:35635)
03:07:47.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:47.012104 13076 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:07:47.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:47.012432 13076 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:07:47.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.005373 12974 master_runner.cc:186] Error getting master registration for 127.12.126.190:35933: OK, instance_id { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" instance_seqno: 1736824062990141 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:07:47.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.040733 12974 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:47.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.272758 12968 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } attempt: 7
03:07:47.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:47.276284 12295 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6260 after lost signal to thread 12227
03:07:47.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:47.277058 12295 debug-util.cc:398] Leaking SignalData structure 0x7b08000c1140 after lost signal to thread 12298
03:07:47.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:47 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824064, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.189@KRBTEST.COM for kudu/127.12.126.188@KRBTEST.COM
03:07:47.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.315507 12968 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343: Bootstrap starting.
03:07:47.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.327977 12968 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343: Neither blocks nor log segments found. Creating new log.
03:07:47.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.331975 12968 log.cc:826] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343: Log is configured to *not* fsync() on all Append() calls
03:07:47.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.340183 12968 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343: No bootstrap required, opened a new log
03:07:47.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.398020 12968 raft_consensus.cc:357] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } }
03:07:47.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.399616 12968 raft_consensus.cc:383] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:07:47.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.400095 12968 raft_consensus.cc:738] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ea2f415b454a4cee864155fdd4bbb343, State: Initialized, Role: FOLLOWER
03:07:47.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.401211 12968 consensus_queue.cc:260] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } }
03:07:47.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.407261 12968 sys_catalog.cc:564] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [sys.catalog]: configured and running, proceeding with master startup.
03:07:47.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.410427 12943 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "16e48fc0bf4746988b005121cc02d88c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ea2f415b454a4cee864155fdd4bbb343" is_pre_election: true
03:07:47.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.413861 12943 raft_consensus.cc:2463] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 16e48fc0bf4746988b005121cc02d88c in term 0.
03:07:47.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.423213 13005 leader_election.cc:304] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 16e48fc0bf4746988b005121cc02d88c, ea2f415b454a4cee864155fdd4bbb343; no voters:
03:07:47.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.424315 13065 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [term 0 FOLLOWER]: Leader pre-election won for term 1
03:07:47.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.424638 13065 raft_consensus.cc:491] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:07:47.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.424949 13065 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [term 0 FOLLOWER]: Advancing to term 1
03:07:47.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.433840 13065 raft_consensus.cc:513] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } }
03:07:47.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.436375 13080 sys_catalog.cc:455] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } } }
03:07:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.448133 12943 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "16e48fc0bf4746988b005121cc02d88c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ea2f415b454a4cee864155fdd4bbb343"
03:07:47.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.449446 12943 raft_consensus.cc:3054] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [term 0 FOLLOWER]: Advancing to term 1
03:07:47.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.450783 13080 sys_catalog.cc:458] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [sys.catalog]: This master's current role is: FOLLOWER
03:07:47.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.451076 13065 leader_election.cc:290] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [CANDIDATE]: Term 1 election: Requested vote from peers b32c0d9df04f4be0b9d7658cf31ee9d4 (127.12.126.190:35933), ea2f415b454a4cee864155fdd4bbb343 (127.12.126.189:35635)
03:07:47.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.468405 12943 raft_consensus.cc:2463] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 16e48fc0bf4746988b005121cc02d88c in term 1.
03:07:47.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.471552 13005 leader_election.cc:304] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 16e48fc0bf4746988b005121cc02d88c, ea2f415b454a4cee864155fdd4bbb343; no voters:
03:07:47.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.472558 13065 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [term 1 FOLLOWER]: Leader election won for term 1
03:07:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.490849 13065 raft_consensus.cc:695] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [term 1 LEADER]: Becoming Leader. State: Replica: 16e48fc0bf4746988b005121cc02d88c, State: Running, Role: LEADER
03:07:47.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.493702 13065 consensus_queue.cc:237] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } }
03:07:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.522122 13087 sys_catalog.cc:455] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 16e48fc0bf4746988b005121cc02d88c. Latest consensus state: current_term: 1 leader_uuid: "16e48fc0bf4746988b005121cc02d88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } } }
03:07:47.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.522891 13087 sys_catalog.cc:458] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [sys.catalog]: This master's current role is: LEADER
03:07:47.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.532243 13092 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
03:07:47.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.540567 13092 catalog_manager.cc:1485] Initializing Kudu cluster ID...
03:07:47.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:47.569146 12922 tablet.cc:2367] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:07:47.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.590929 12943 raft_consensus.cc:1270] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [term 1 FOLLOWER]: Refusing update from remote peer 16e48fc0bf4746988b005121cc02d88c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:07:47.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.593287 13065 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:07:47.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:47.594772 13095 catalog_manager.cc:1559] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:07:47.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:47.595263 13095 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:07:47.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.554136 12896 master_runner.cc:186] Error getting master registration for 127.12.126.190:35933: OK, instance_id { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" instance_seqno: 1736824062990141 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:07:47.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.632184 12896 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:47.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.644068 13080 sys_catalog.cc:455] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 16e48fc0bf4746988b005121cc02d88c. Latest consensus state: current_term: 1 leader_uuid: "16e48fc0bf4746988b005121cc02d88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } } }
03:07:47.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.644835 13080 sys_catalog.cc:458] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [sys.catalog]: This master's current role is: FOLLOWER
03:07:47.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.647156 13096 mvcc.cc:204] Tried to move back new op lower bound from 7114031380798386176 to 7114031380484599808. Current Snapshot: MvccSnapshot[applied={T|T < 7114031380798386176}]
03:07:47.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.653118 13098 mvcc.cc:204] Tried to move back new op lower bound from 7114031380798386176 to 7114031380484599808. Current Snapshot: MvccSnapshot[applied={T|T < 7114031380798386176}]
03:07:47.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.658192 13080 sys_catalog.cc:455] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "16e48fc0bf4746988b005121cc02d88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } } }
03:07:47.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.658932 13080 sys_catalog.cc:458] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [sys.catalog]: This master's current role is: FOLLOWER
03:07:47.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.659977 13087 sys_catalog.cc:455] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "16e48fc0bf4746988b005121cc02d88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } } }
03:07:47.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.660784 13087 sys_catalog.cc:458] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [sys.catalog]: This master's current role is: LEADER
03:07:47.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.661916 13092 catalog_manager.cc:1348] Generated new cluster ID: fc0b2ecc8c564017acad54664a84693a
03:07:47.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.662204 13092 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
03:07:47.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.677368 12890 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } attempt: 7
03:07:47.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:47 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824062, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.190@KRBTEST.COM for kudu/127.12.126.188@KRBTEST.COM
03:07:47.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.726699 13092 catalog_manager.cc:1371] Generated new certificate authority record
03:07:47.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.729247 13092 catalog_manager.cc:1505] Loading token signing keys...
03:07:47.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.748021 12890 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4: Bootstrap starting.
03:07:47.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.766383 13092 catalog_manager.cc:5899] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c: Generated new TSK 0
03:07:47.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.769122 13092 catalog_manager.cc:1515] Initializing in-progress tserver states...
03:07:47.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.786088 12890 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4: Neither blocks nor log segments found. Creating new log.
03:07:47.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.793262 12890 log.cc:826] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4: Log is configured to *not* fsync() on all Append() calls
03:07:47.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.837333 12890 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4: No bootstrap required, opened a new log
03:07:47.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.126.129@KRBTEST.COM; defaulting to no policy
03:07:47.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:47.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.126.129@KRBTEST.COM" created.
03:07:47.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:47.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.126.129 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.129.keytab.
03:07:47.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.126.129 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.129.keytab.
03:07:47.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:47.991868 12794 mini_kdc.cc:262] Time spent creating service keytab for kudu/127.12.126.129: real 0.178s user 0.004s sys 0.012s
03:07:47.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:47.994135 12794 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:47.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:47.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-0/wal
03:07:47.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-0/data
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-0/logs
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-0/data/info.pb
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.126.129:0
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.126.129
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.126.129
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.129.keytab
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.126.129
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.126.148:44839
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {KRB5CCNAME=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:48.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.006618 12890 raft_consensus.cc:357] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } }
03:07:48.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.014019 12890 raft_consensus.cc:383] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:07:48.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.015731 12890 raft_consensus.cc:738] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b32c0d9df04f4be0b9d7658cf31ee9d4, State: Initialized, Role: FOLLOWER
03:07:48.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.021024 12890 consensus_queue.cc:260] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } }
03:07:48.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.058531 13107 sys_catalog.cc:455] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } } }
03:07:48.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.065023 12865 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "16e48fc0bf4746988b005121cc02d88c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" is_pre_election: true
03:07:48.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.065928 12865 raft_consensus.cc:2463] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 16e48fc0bf4746988b005121cc02d88c in term 0.
03:07:48.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.066423 12865 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:33868 (request call id 2) took 1058 ms. Trace:
03:07:48.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.066728 12865 rpcz_store.cc:276] 0114 03:07:47.007855 (+ 0us) service_pool.cc:168] Inserting onto call queue
03:07:48.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:07:47.007981 (+ 126us) service_pool.cc:225] Handling call
03:07:48.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:07:48.066369 (+1058388us) inbound_call.cc:173] Queueing success response
03:07:48.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
03:07:48.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.068684 12863 raft_consensus.cc:3054] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [term 0 FOLLOWER]: Advancing to term 1
03:07:48.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.071733 12864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "16e48fc0bf4746988b005121cc02d88c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4"
03:07:48.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.080155 12863 raft_consensus.cc:1270] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [term 1 FOLLOWER]: Refusing update from remote peer 16e48fc0bf4746988b005121cc02d88c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:07:48.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.081256 13107 sys_catalog.cc:458] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [sys.catalog]: This master's current role is: FOLLOWER
03:07:48.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.081995 13065 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [LEADER]: Connected to new peer: Peer: permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
03:07:48.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.082139 12890 sys_catalog.cc:564] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [sys.catalog]: configured and running, proceeding with master startup.
03:07:48.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.212769 13116 sys_catalog.cc:455] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "16e48fc0bf4746988b005121cc02d88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } } }
03:07:48.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.214509 13116 sys_catalog.cc:458] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [sys.catalog]: This master's current role is: FOLLOWER
03:07:48.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.211117 13107 sys_catalog.cc:455] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 16e48fc0bf4746988b005121cc02d88c. Latest consensus state: current_term: 1 leader_uuid: "16e48fc0bf4746988b005121cc02d88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } } }
03:07:48.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.215397 13107 sys_catalog.cc:458] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [sys.catalog]: This master's current role is: FOLLOWER
03:07:48.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.244706 13124 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
03:07:48.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.510519 13087 sys_catalog.cc:455] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "16e48fc0bf4746988b005121cc02d88c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } } }
03:07:48.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.511282 13087 sys_catalog.cc:458] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [sys.catalog]: This master's current role is: LEADER
03:07:48.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.600890 13095 catalog_manager.cc:1260] Loaded cluster ID: fc0b2ecc8c564017acad54664a84693a
03:07:48.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.601527 13095 catalog_manager.cc:1553] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343: loading cluster ID for follower catalog manager: success
03:07:48.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.614971 13095 catalog_manager.cc:1575] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343: acquiring CA information for follower catalog manager: success
03:07:48.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.628439 13095 catalog_manager.cc:1603] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:48.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:48.704993 13106 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:48.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:48.705897 13106 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:48.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:48.707047 13106 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:48.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:48.767259 13106 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:48.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:48.769119 13106 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.126.129
03:07:48.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.848394 13106 tablet_server_runner.cc:78] Tablet server non-default flags:
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.126.148:44839
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-0/data
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-0/wal
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.129.keytab
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.126.129
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.126.129:0
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-0/data/info.pb
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.126.129
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.126.129
03:07:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-0/logs
03:07:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:07:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:07:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
03:07:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 02:43:23 UTC on a633016ff5c9
03:07:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4168
03:07:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:48.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.851918 13106 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:48.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:48.853467 13106 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:48.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:48.898690 13132 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:49.272204 13124 catalog_manager.cc:1260] Loaded cluster ID: fc0b2ecc8c564017acad54664a84693a
03:07:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:49.272527 13124 catalog_manager.cc:1553] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4: loading cluster ID for follower catalog manager: success
03:07:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:49.311110 13124 catalog_manager.cc:1575] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4: acquiring CA information for follower catalog manager: success
03:07:49.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:49.337903 13124 catalog_manager.cc:1603] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:48.909226 13135 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:51.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:48.903577 13131 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:51.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.564327 13106 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:51.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:51.564988 13133 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2663 milliseconds
03:07:51.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.566565 13106 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:51.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.569469 13106 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:51.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.571025 13106 hybrid_clock.cc:648] HybridClock initialized: now 1736824071570990 us; error 43 us; skew 500 ppm
03:07:51.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.572126 13106 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:51.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:51 dist-test-slave-rqxq krb5kdc[12807](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824071, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.129@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:51.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.587172 13106 init.cc:377] Logged in from keytab as kudu/127.12.126.129@KRBTEST.COM (short username kudu)
03:07:51.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.593935 13106 webserver.cc:458] Webserver started at http://127.12.126.129:46701/ using document root <none> and password file <none>
03:07:51.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.595249 13106 fs_manager.cc:362] Metadata directory not provided
03:07:51.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.595557 13106 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:51.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.596150 13106 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:07:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.603595 13106 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-0/data/instance:
03:07:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "85d38da4c73c440db25182354e328654"
03:07:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:51 on dist-test-slave-rqxq"
03:07:51.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.605053 13106 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-0/wal/instance:
03:07:51.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "85d38da4c73c440db25182354e328654"
03:07:51.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:51 on dist-test-slave-rqxq"
03:07:51.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.614095 13106 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.003s
03:07:51.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.620179 13147 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:51.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.621347 13106 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
03:07:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.621683 13106 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-0/data,/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-0/wal
03:07:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "85d38da4c73c440db25182354e328654"
03:07:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:51 on dist-test-slave-rqxq"
03:07:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.622009 13106 fs_report.cc:389] FS layout report
03:07:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-0/wal
03:07:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-0/wal
03:07:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-0/data/data
03:07:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:51.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:51.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:51.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.686811 13106 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:51.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.693146 13106 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:51.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.693684 13106 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:51.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.697258 13106 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:07:51.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.717146 13106 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:07:51.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.717430 13106 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:07:51.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.717736 13106 ts_tablet_manager.cc:610] Registered 0 tablets
03:07:51.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:51.717986 13106 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:07:52.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:52.110553 13106 rpc_server.cc:307] RPC server started. Bound to: 127.12.126.129:43981
03:07:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:52.110651 13261 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.126.129:43981 every 8 connection(s)
03:07:52.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:52.114223 13106 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-0/data/info.pb
03:07:52.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:52.124753 12794 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu as pid 13106
03:07:52.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:52.125385 12794 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-0/wal/instance
03:07:52.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:52 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824071, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.129@KRBTEST.COM for kudu/127.12.126.189@KRBTEST.COM
03:07:52.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.126.130@KRBTEST.COM; defaulting to no policy
03:07:52.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:52.217438 13262 heartbeater.cc:346] Connected to a master server at 127.12.126.189:35635
03:07:52.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:52.217985 13262 heartbeater.cc:463] Registering TS with master...
03:07:52.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:52.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.126.130@KRBTEST.COM" created.
03:07:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:52.223482 13262 heartbeater.cc:510] Master 127.12.126.189:35635 requested a full tablet report, sending...
03:07:52.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:52.227082 12933 ts_manager.cc:194] Registered new tserver with Master: 85d38da4c73c440db25182354e328654 (127.12.126.129:43981)
03:07:52.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:52 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824071, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.129@KRBTEST.COM for kudu/127.12.126.190@KRBTEST.COM
03:07:52.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:52 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824071, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.129@KRBTEST.COM for kudu/127.12.126.188@KRBTEST.COM
03:07:52.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:52.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.126.130 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.130.keytab.
03:07:52.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.126.130 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.130.keytab.
03:07:52.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:52.272586 12794 mini_kdc.cc:262] Time spent creating service keytab for kudu/127.12.126.130: real 0.132s user 0.002s sys 0.010s
03:07:52.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:52.274969 12794 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:52.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:52.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-1/wal
03:07:52.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-1/data
03:07:52.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:52.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-1/logs
03:07:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-1/data/info.pb
03:07:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:07:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.126.130:0
03:07:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.126.130
03:07:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.126.130
03:07:52.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:52.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957
03:07:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.130.keytab
03:07:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.126.130
03:07:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.126.148:44839
03:07:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {KRB5CCNAME=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:52.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:52.300176 13267 heartbeater.cc:346] Connected to a master server at 127.12.126.190:35933
03:07:52.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:52.300556 13267 heartbeater.cc:463] Registering TS with master...
03:07:52.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:52.301509 13267 heartbeater.cc:510] Master 127.12.126.190:35933 requested a full tablet report, sending...
03:07:52.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:52.304028 13263 heartbeater.cc:346] Connected to a master server at 127.12.126.188:40957
03:07:52.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:52.304364 13263 heartbeater.cc:463] Registering TS with master...
03:07:52.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:52.305352 13263 heartbeater.cc:510] Master 127.12.126.188:40957 requested a full tablet report, sending...
03:07:52.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:52.305511 12855 ts_manager.cc:194] Registered new tserver with Master: 85d38da4c73c440db25182354e328654 (127.12.126.129:43981)
03:07:52.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:52.309551 13019 ts_manager.cc:194] Registered new tserver with Master: 85d38da4c73c440db25182354e328654 (127.12.126.129:43981)
03:07:52.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:52.312323 13019 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.12.126.129@KRBTEST.COM'} at 127.12.126.129:41769
03:07:52.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:52.654515 13274 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:52.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:52.655046 13274 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:52.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:52.655965 13274 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:52.689646 13274 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:52.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:52.690757 13274 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.126.130
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:52.739589 13274 tablet_server_runner.cc:78] Tablet server non-default flags:
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.126.148:44839
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-1/data
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-1/wal
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.130.keytab
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.126.130
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.126.130:0
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-1/data/info.pb
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.126.130
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.126.130
03:07:52.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-1/logs
03:07:52.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:52.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:52.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:52.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:07:52.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:07:52.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
03:07:52.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:52.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 02:43:23 UTC on a633016ff5c9
03:07:52.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4168
03:07:52.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:52.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:52.741210 13274 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:52.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:52.742436 13274 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:52.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:52.763976 13281 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:52.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:52 dist-test-slave-rqxq krb5kdc[12807](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824072, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:52.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:52.863914 12834 init.cc:295] Successfully reacquired a new kerberos TGT
03:07:53.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:53.317850 13263 heartbeater.cc:502] Master 127.12.126.188:40957 was elected leader, sending a full tablet report...
03:07:54.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:54.159931 13280 debug-util.cc:398] Leaking SignalData structure 0x7b080001ca40 after lost signal to thread 13274
03:07:54.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:52.764886 13282 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:54.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:54.202916 13274 thread.cc:635] GCE (cloud detector) Time spent creating pthread: real 1.440s user 0.521s sys 0.864s
03:07:54.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:54.203320 13274 thread.cc:601] GCE (cloud detector) Time spent starting thread: real 1.441s user 0.521s sys 0.864s
03:07:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.213920 13274 server_base.cc:1034] running on GCE node
03:07:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:54.214676 13289 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:54.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.215817 13274 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:54.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.217839 13274 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:54.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.219228 13274 hybrid_clock.cc:648] HybridClock initialized: now 1736824074219179 us; error 45 us; skew 500 ppm
03:07:54.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.220017 13274 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:54.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:54 dist-test-slave-rqxq krb5kdc[12807](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824074, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.130@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:54.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.226094 13274 init.cc:377] Logged in from keytab as kudu/127.12.126.130@KRBTEST.COM (short username kudu)
03:07:54.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.230562 13274 webserver.cc:458] Webserver started at http://127.12.126.130:42313/ using document root <none> and password file <none>
03:07:54.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.231400 13274 fs_manager.cc:362] Metadata directory not provided
03:07:54.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.231623 13274 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:54.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.231989 13274 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:07:54.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.237434 13274 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-1/data/instance:
03:07:54.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5aa517d245cb498d8bc4c31b9f847d9e"
03:07:54.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:54 on dist-test-slave-rqxq"
03:07:54.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.238461 13274 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-1/wal/instance:
03:07:54.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5aa517d245cb498d8bc4c31b9f847d9e"
03:07:54.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:54 on dist-test-slave-rqxq"
03:07:54.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.245322 13274 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.000s sys 0.008s
03:07:54.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.250475 13296 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:54.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.251426 13274 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
03:07:54.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.251783 13274 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-1/data,/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-1/wal
03:07:54.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5aa517d245cb498d8bc4c31b9f847d9e"
03:07:54.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:54 on dist-test-slave-rqxq"
03:07:54.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.252105 13274 fs_report.cc:389] FS layout report
03:07:54.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:54.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-1/wal
03:07:54.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-1/wal
03:07:54.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-1/data/data
03:07:54.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:54.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:54.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:54.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:54.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:54.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:54.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:54.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:54.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:54.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:54.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:54.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.310711 13274 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:54.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.318960 13274 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:54.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.319428 13274 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:54.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.321858 13274 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:07:54.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.326524 13274 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:07:54.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.326710 13274 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:07:54.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.326908 13274 ts_tablet_manager.cc:610] Registered 0 tablets
03:07:54.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.327054 13274 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:07:54.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.479424 13274 rpc_server.cc:307] RPC server started. Bound to: 127.12.126.130:34537
03:07:54.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.479601 13408 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.126.130:34537 every 8 connection(s)
03:07:54.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.481966 13274 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-1/data/info.pb
03:07:54.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.489917 12794 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu as pid 13274
03:07:54.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.490478 12794 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-1/wal/instance
03:07:54.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.126.131@KRBTEST.COM; defaulting to no policy
03:07:54.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:54.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.126.131@KRBTEST.COM" created.
03:07:54.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:54 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824074, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.130@KRBTEST.COM for kudu/127.12.126.188@KRBTEST.COM
03:07:54.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:54 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824074, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.130@KRBTEST.COM for kudu/127.12.126.190@KRBTEST.COM
03:07:54.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:54 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824074, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.130@KRBTEST.COM for kudu/127.12.126.189@KRBTEST.COM
03:07:54.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.591233 13410 heartbeater.cc:346] Connected to a master server at 127.12.126.188:40957
03:07:54.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.591979 13410 heartbeater.cc:463] Registering TS with master...
03:07:54.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.594036 13410 heartbeater.cc:510] Master 127.12.126.188:40957 requested a full tablet report, sending...
03:07:54.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.596793 13412 heartbeater.cc:346] Connected to a master server at 127.12.126.190:35933
03:07:54.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.597404 13412 heartbeater.cc:463] Registering TS with master...
03:07:54.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:54.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.126.131 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.131.keytab.
03:07:54.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.126.131 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.131.keytab.
03:07:54.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.598206 13019 ts_manager.cc:194] Registered new tserver with Master: 5aa517d245cb498d8bc4c31b9f847d9e (127.12.126.130:34537)
03:07:54.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.598896 13412 heartbeater.cc:510] Master 127.12.126.190:35933 requested a full tablet report, sending...
03:07:54.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.600744 13019 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.12.126.130@KRBTEST.COM'} at 127.12.126.130:49903
03:07:54.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.602310 12794 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:54.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:54.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-2/wal
03:07:54.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-2/data
03:07:54.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:54.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:54.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:54.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:54.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:54.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:54.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-2/logs
03:07:54.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-2/data/info.pb
03:07:54.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:54.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:54.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:54.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:54.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:54.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:07:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.126.131:0
03:07:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.126.131
03:07:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.126.131
03:07:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957
03:07:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.131.keytab
03:07:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.126.131
03:07:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.126.148:44839
03:07:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:54.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {KRB5CCNAME=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:54.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.609014 13409 heartbeater.cc:346] Connected to a master server at 127.12.126.189:35635
03:07:54.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.609133 12855 ts_manager.cc:194] Registered new tserver with Master: 5aa517d245cb498d8bc4c31b9f847d9e (127.12.126.130:34537)
03:07:54.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.609481 13409 heartbeater.cc:463] Registering TS with master...
03:07:54.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.610553 13409 heartbeater.cc:510] Master 127.12.126.189:35635 requested a full tablet report, sending...
03:07:54.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.613924 12933 ts_manager.cc:194] Registered new tserver with Master: 5aa517d245cb498d8bc4c31b9f847d9e (127.12.126.130:34537)
03:07:54.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:54 dist-test-slave-rqxq krb5kdc[12807](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824074, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:54.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:54.716851 12912 init.cc:295] Successfully reacquired a new kerberos TGT
03:07:54.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:54.758819 13258 debug-util.cc:398] Leaking SignalData structure 0x7b08000b20c0 after lost signal to thread 13126
03:07:54.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:54.760471 13258 debug-util.cc:398] Leaking SignalData structure 0x7b080003cf40 after lost signal to thread 13261
03:07:55.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:55.032294 13421 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:55.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:55.032922 13421 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:55.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:55.033881 13421 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:55.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:55.086860 13421 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:55.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:55.087810 13421 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.126.131
03:07:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:55.125952 13421 tablet_server_runner.cc:78] Tablet server non-default flags:
03:07:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.126.148:44839
03:07:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-2/data
03:07:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-2/wal
03:07:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.131.keytab
03:07:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.126.131
03:07:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.126.131:0
03:07:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:55.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:55.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-2/data/info.pb
03:07:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.126.131
03:07:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957
03:07:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.126.131
03:07:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-2/logs
03:07:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:07:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:07:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
03:07:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 02:43:23 UTC on a633016ff5c9
03:07:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4168
03:07:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:55.127236 13421 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:55.128393 13421 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:55.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:55.147830 13427 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:55.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:55.605823 13410 heartbeater.cc:502] Master 127.12.126.188:40957 was elected leader, sending a full tablet report...
03:07:56.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:55.150877 13421 server_base.cc:1034] running on GCE node
03:07:56.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:55.149684 13430 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:56.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:55.148947 13428 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:56.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.422930 13421 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.425899 13421 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:56.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.427467 13421 hybrid_clock.cc:648] HybridClock initialized: now 1736824076427413 us; error 52 us; skew 500 ppm
03:07:56.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.428604 13421 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:56.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:56 dist-test-slave-rqxq krb5kdc[12807](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824076, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.131@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:56.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.434568 13421 init.cc:377] Logged in from keytab as kudu/127.12.126.131@KRBTEST.COM (short username kudu)
03:07:56.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.440006 13421 webserver.cc:458] Webserver started at http://127.12.126.131:35139/ using document root <none> and password file <none>
03:07:56.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.440958 13421 fs_manager.cc:362] Metadata directory not provided
03:07:56.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.441198 13421 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:56.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.441663 13421 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:07:56.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.446708 13421 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-2/data/instance:
03:07:56.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dcb2a708806e4ce0a45e62f438922aa5"
03:07:56.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:56 on dist-test-slave-rqxq"
03:07:56.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.447904 13421 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-2/wal/instance:
03:07:56.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dcb2a708806e4ce0a45e62f438922aa5"
03:07:56.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:56 on dist-test-slave-rqxq"
03:07:56.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.455746 13421 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
03:07:56.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.462033 13445 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:56.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.463145 13421 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
03:07:56.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.463471 13421 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-2/data,/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-2/wal
03:07:56.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dcb2a708806e4ce0a45e62f438922aa5"
03:07:56.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:56 on dist-test-slave-rqxq"
03:07:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.463857 13421 fs_report.cc:389] FS layout report
03:07:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-2/wal
03:07:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-2/wal
03:07:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-2/data/data
03:07:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:56.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:56.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:56.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.527645 13421 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:56.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.534013 13421 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:56.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.534500 13421 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:56.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.537551 13421 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:07:56.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.542620 13421 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:07:56.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.542835 13421 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:07:56.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.543154 13421 ts_tablet_manager.cc:610] Registered 0 tablets
03:07:56.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.543326 13421 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:07:56.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:56 dist-test-slave-rqxq krb5kdc[12807](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824076, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:56.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.652297 12998 init.cc:295] Successfully reacquired a new kerberos TGT
03:07:56.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.743746 13421 rpc_server.cc:307] RPC server started. Bound to: 127.12.126.131:38289
03:07:56.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.743892 13557 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.126.131:38289 every 8 connection(s)
03:07:56.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.746505 13421 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-2/data/info.pb
03:07:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.757582 12794 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu as pid 13421
03:07:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.758201 12794 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/ts-2/wal/instance
03:07:56.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:56 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824076, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.131@KRBTEST.COM for kudu/127.12.126.189@KRBTEST.COM
03:07:56.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:56 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824076, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.131@KRBTEST.COM for kudu/127.12.126.190@KRBTEST.COM
03:07:56.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:56 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824076, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.131@KRBTEST.COM for kudu/127.12.126.188@KRBTEST.COM
03:07:56.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.818501 13560 heartbeater.cc:346] Connected to a master server at 127.12.126.190:35933
03:07:56.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.818881 13558 heartbeater.cc:346] Connected to a master server at 127.12.126.189:35635
03:07:56.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.818984 13560 heartbeater.cc:463] Registering TS with master...
03:07:56.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.819137 13558 heartbeater.cc:463] Registering TS with master...
03:07:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.820436 13558 heartbeater.cc:510] Master 127.12.126.189:35635 requested a full tablet report, sending...
03:07:56.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.820428 13560 heartbeater.cc:510] Master 127.12.126.190:35933 requested a full tablet report, sending...
03:07:56.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.824846 12932 ts_manager.cc:194] Registered new tserver with Master: dcb2a708806e4ce0a45e62f438922aa5 (127.12.126.131:38289)
03:07:56.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.826295 12855 ts_manager.cc:194] Registered new tserver with Master: dcb2a708806e4ce0a45e62f438922aa5 (127.12.126.131:38289)
03:07:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.828305 13559 heartbeater.cc:346] Connected to a master server at 127.12.126.188:40957
03:07:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.828639 13559 heartbeater.cc:463] Registering TS with master...
03:07:56.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.829401 13559 heartbeater.cc:510] Master 127.12.126.188:40957 requested a full tablet report, sending...
03:07:56.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.831036 13019 ts_manager.cc:194] Registered new tserver with Master: dcb2a708806e4ce0a45e62f438922aa5 (127.12.126.131:38289)
03:07:56.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.832567 13019 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.12.126.131@KRBTEST.COM'} at 127.12.126.131:44663
03:07:56.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.844111 12794 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
03:07:56.856 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:56.857 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
03:07:56.866 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
03:07:56.866 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
03:07:56.869 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.12.126.190"
port: 35933
}
bound_http_address {
host: "127.12.126.190"
port: 42053
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.12.126.189"
port: 35635
}
bound_http_address {
host: "127.12.126.189"
port: 38527
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.12.126.188"
port: 40957
}
bound_http_address {
host: "127.12.126.188"
port: 35901
}
}
}
03:07:56.869 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:07:56.870 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.12.126.129"
port: 43981
}
bound_http_address {
host: "127.12.126.129"
port: 46701
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.12.126.130"
port: 34537
}
bound_http_address {
host: "127.12.126.130"
port: 42313
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.12.126.131"
port: 38289
}
bound_http_address {
host: "127.12.126.131"
port: 35139
}
}
}
03:07:56.870 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:07:56.871 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:07:56.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:56 dist-test-slave-rqxq krb5kdc[12807](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824076, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:56.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:07:56.903 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:56.905 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/krb5cc
03:07:56.913 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:07:56.914 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/krb5cc
03:07:56.915 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:07:56.919 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
03:07:56.921 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:07:56.921 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:07:56.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:56 dist-test-slave-rqxq krb5kdc[12807](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824076, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:56.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:07:56.951 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:56.952 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/krb5cc
03:07:56.953 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:07:56.953 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.126.190:35933
03:07:56.955 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
03:07:56.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.955829 12794 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu with pid 12820
03:07:56.983 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:56.984 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.126.189:35635
03:07:56.984 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
03:07:56.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:56.984769 12794 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu with pid 12896
03:07:56.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:56.996134 13003 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.126.190:35933: connect: Connection refused (error 111)
03:07:57.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:56.999742 13003 consensus_peers.cc:487] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c -> Peer b32c0d9df04f4be0b9d7658cf31ee9d4 (127.12.126.190:35933): Couldn't send request to peer b32c0d9df04f4be0b9d7658cf31ee9d4. Status: Network error: Client connection negotiation failed: client connection to 127.12.126.190:35933: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:07:57.014 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:57.014 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.126.188:40957
03:07:57.014 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:07:57.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:57.014770 12794 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu with pid 12974
03:07:57.040 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:57.040 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.126.190:35933
03:07:57.043 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
03:07:57.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:57.045238 12794 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:57.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/wal
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/data
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/logs
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/data/info.pb
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.126.190:35933
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.126.190
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42053
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.190.keytab
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.126.190
03:07:57.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:57.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:57.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:57.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.126.148:44839
03:07:57.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:57.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:57.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:57.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:57.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957
03:07:57.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:57.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34021/jwks.json
03:07:57.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:57.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:57.246822 13262 heartbeater.cc:643] Failed to heartbeat to 127.12.126.189:35635 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.12.126.189:35635: connect: Connection refused (error 111)
03:07:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:57.369488 13575 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:57.370035 13575 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:57.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:57.370662 13575 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:57.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:57.401479 13575 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:57.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:57.401712 13575 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:57.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:57.401981 13575 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:57.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:57.402221 13575 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:57.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:57.402413 13575 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:57.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:57.402597 13575 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:57.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:57.402738 13575 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34021/jwks.json
03:07:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:57.439497 13575 master_runner.cc:386] Master server non-default flags:
03:07:57.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:57.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.126.148:44839
03:07:57.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:57.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:57.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/data
03:07:57.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/wal
03:07:57.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:57.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:57.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957
03:07:57.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.190.keytab
03:07:57.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.126.190
03:07:57.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.126.190:35933
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34021/jwks.json
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/data/info.pb
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.126.190
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42053
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/logs
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 02:43:23 UTC on a633016ff5c9
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4168
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:57.440747 13575 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:57.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:57.441692 13575 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:57.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:57.457856 13582 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:57.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:57.619134 13410 heartbeater.cc:643] Failed to heartbeat to 127.12.126.188:40957 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.12.126.188:40957: connect: Connection refused (error 111)
03:07:57.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:57.833707 13560 heartbeater.cc:643] Failed to heartbeat to 127.12.126.190:35933 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.12.126.190:35933: connect: Connection refused (error 111)
03:07:57.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:57.836486 13559 heartbeater.cc:502] Master 127.12.126.188:40957 was elected leader, sending a full tablet report...
03:07:58.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:57.458705 13583 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:58.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:57.459456 13585 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:58.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:57.459846 13575 server_base.cc:1034] running on GCE node
03:07:58.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.700148 13575 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:58.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.703454 13575 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:58.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.704928 13575 hybrid_clock.cc:648] HybridClock initialized: now 1736824078704885 us; error 42 us; skew 500 ppm
03:07:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.705983 13575 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:58.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:58 dist-test-slave-rqxq krb5kdc[12807](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824078, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:58.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.713891 13575 init.cc:377] Logged in from keytab as kudu/127.12.126.190@KRBTEST.COM (short username kudu)
03:07:58.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.719497 13575 webserver.cc:458] Webserver started at http://127.12.126.190:42053/ using document root <none> and password file <none>
03:07:58.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.720737 13575 fs_manager.cc:362] Metadata directory not provided
03:07:58.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.721016 13575 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:58.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.732651 13575 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.006s sys 0.001s
03:07:58.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.738106 13603 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:58.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.739346 13575 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
03:07:58.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.739751 13575 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/data,/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/wal
03:07:58.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4"
03:07:58.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:42 on dist-test-slave-rqxq"
03:07:58.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.743006 13575 fs_report.cc:389] FS layout report
03:07:58.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:58.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/wal
03:07:58.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/wal
03:07:58.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/data/data
03:07:58.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:58.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:58.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:58.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:58.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:58.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:58.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:58.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:58.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:58.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:58.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:58.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.800218 13575 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:58.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.805521 13575 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:58.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.806088 13575 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:58.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.878615 13575 rpc_server.cc:307] RPC server started. Bound to: 127.12.126.190:35933
03:07:58.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.878726 13657 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.126.190:35933 every 8 connection(s)
03:07:58.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.881489 13575 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-0/data/info.pb
03:07:58.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.889102 12794 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu as pid 13575
03:07:58.889 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:58.889 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.126.189:35635
03:07:58.890 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
03:07:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.891340 12794 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:07:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/wal
03:07:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/data
03:07:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/logs
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/data/info.pb
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.126.189:35635
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.126.189
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38527
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.189.keytab
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.126.189
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:58.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.126.148:44839
03:07:58.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:58.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:58.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:58.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:58.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957
03:07:58.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:58.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34021/jwks.json
03:07:58.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:58.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.892046 13658 sys_catalog.cc:263] Verifying existing consensus state
03:07:58.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.898594 13658 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4: Bootstrap starting.
03:07:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.955782 13658 log.cc:826] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4: Log is configured to *not* fsync() on all Append() calls
03:07:58.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.974386 13658 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:07:58.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:58.975350 13658 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4: Bootstrap complete.
03:07:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.000727 13658 raft_consensus.cc:357] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } }
03:07:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.003121 13658 raft_consensus.cc:738] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: b32c0d9df04f4be0b9d7658cf31ee9d4, State: Initialized, Role: FOLLOWER
03:07:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.003916 13658 consensus_queue.cc:260] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } }
03:07:59.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.006716 13662 sys_catalog.cc:455] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } } }
03:07:59.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.007750 13662 sys_catalog.cc:458] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [sys.catalog]: This master's current role is: FOLLOWER
03:07:59.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.008584 13658 sys_catalog.cc:564] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [sys.catalog]: configured and running, proceeding with master startup.
03:07:59.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.044898 13673 catalog_manager.cc:1260] Loaded cluster ID: fc0b2ecc8c564017acad54664a84693a
03:07:59.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.045199 13673 catalog_manager.cc:1553] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4: loading cluster ID for follower catalog manager: success
03:07:59.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.053289 13673 catalog_manager.cc:1575] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4: acquiring CA information for follower catalog manager: success
03:07:59.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:07:59 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824078, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.190@KRBTEST.COM for kudu/127.12.126.190@KRBTEST.COM
03:07:59.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.073138 13673 catalog_manager.cc:1603] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:59.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.078521 13575 master_runner.cc:186] Error getting master registration for 127.12.126.189:35635: Network error: Client connection negotiation failed: client connection to 127.12.126.189:35635: connect: Connection refused (error 111),
03:07:59.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.098434 13575 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:59.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:59.296675 13660 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:59.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:59.297339 13660 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:59.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:59.298060 13660 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:59.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:59.332469 13660 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:59.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:59.332810 13660 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:59.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:59.333307 13660 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:59.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:59.333662 13660 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:59.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:59.334000 13660 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:59.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:59.334347 13660 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:59.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:59.334636 13660 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34021/jwks.json
03:07:59.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.346287 13267 heartbeater.cc:346] Connected to a master server at 127.12.126.190:35933
03:07:59.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.360236 13620 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "85d38da4c73c440db25182354e328654" instance_seqno: 1736824071986688) as {username='kudu', principal='kudu/127.12.126.129@KRBTEST.COM'} at 127.12.126.129:34353; Asking this server to re-register.
03:07:59.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.362371 13267 heartbeater.cc:463] Registering TS with master...
03:07:59.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.365753 13620 ts_manager.cc:194] Registered new tserver with Master: 85d38da4c73c440db25182354e328654 (127.12.126.129:43981)
03:07:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.386744 13660 master_runner.cc:386] Master server non-default flags:
03:07:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.126.148:44839
03:07:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/data
03:07:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/wal
03:07:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957
03:07:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.189.keytab
03:07:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.126.189
03:07:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.126.189:35635
03:07:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34021/jwks.json
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/data/info.pb
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.126.189
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38527
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/logs
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 02:43:23 UTC on a633016ff5c9
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4168
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.388465 13660 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:59.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.389649 13660 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:59.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:59.410656 13686 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:59.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.668949 13412 heartbeater.cc:346] Connected to a master server at 127.12.126.190:35933
03:07:59.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.672152 13620 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "5aa517d245cb498d8bc4c31b9f847d9e" instance_seqno: 1736824074444003) as {username='kudu', principal='kudu/127.12.126.130@KRBTEST.COM'} at 127.12.126.130:50171; Asking this server to re-register.
03:07:59.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.673616 13412 heartbeater.cc:463] Registering TS with master...
03:07:59.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.677978 13620 ts_manager.cc:194] Registered new tserver with Master: 5aa517d245cb498d8bc4c31b9f847d9e (127.12.126.130:34537)
03:07:59.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.908368 13560 heartbeater.cc:346] Connected to a master server at 127.12.126.190:35933
03:07:59.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.911270 13620 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "dcb2a708806e4ce0a45e62f438922aa5" instance_seqno: 1736824076698615) as {username='kudu', principal='kudu/127.12.126.131@KRBTEST.COM'} at 127.12.126.131:39443; Asking this server to re-register.
03:07:59.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.912887 13560 heartbeater.cc:463] Registering TS with master...
03:07:59.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:07:59.916775 13620 ts_manager.cc:194] Registered new tserver with Master: dcb2a708806e4ce0a45e62f438922aa5 (127.12.126.131:38289)
03:07:59.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:59.949280 12141 debug-util.cc:398] Leaking SignalData structure 0x7b080009e1a0 after lost signal to thread 12078
03:07:59.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:59.950237 12141 debug-util.cc:398] Leaking SignalData structure 0x7b08000b23e0 after lost signal to thread 12144
03:08:00.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:00.776842 13554 debug-util.cc:398] Leaking SignalData structure 0x7b08000bf420 after lost signal to thread 13422
03:08:00.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:00.778156 13554 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba380 after lost signal to thread 13557
03:08:00.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:00.787338 13705 raft_consensus.cc:491] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:08:00.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:00.788126 13705 raft_consensus.cc:513] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } }
03:08:00.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:00.797881 13606 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.126.189:35635: connect: Connection refused (error 111)
03:08:00.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:00.802335 13705 leader_election.cc:290] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers ea2f415b454a4cee864155fdd4bbb343 (127.12.126.189:35635), 16e48fc0bf4746988b005121cc02d88c (127.12.126.188:40957)
03:08:00.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:00.814224 13607 leader_election.cc:336] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 16e48fc0bf4746988b005121cc02d88c (127.12.126.188:40957): Network error: Client connection negotiation failed: client connection to 127.12.126.188:40957: connect: Connection refused (error 111)
03:08:00.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:00.824080 13606 leader_election.cc:336] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer ea2f415b454a4cee864155fdd4bbb343 (127.12.126.189:35635): Network error: Client connection negotiation failed: client connection to 127.12.126.189:35635: connect: Connection refused (error 111)
03:08:00.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:00.824553 13606 leader_election.cc:304] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b32c0d9df04f4be0b9d7658cf31ee9d4; no voters: 16e48fc0bf4746988b005121cc02d88c, ea2f415b454a4cee864155fdd4bbb343
03:08:00.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:00.825445 13705 raft_consensus.cc:2743] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
03:08:01.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:07:59.412632 13685 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:01.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:00.803843 13684 debug-util.cc:398] Leaking SignalData structure 0x7b080001cac0 after lost signal to thread 13660
03:08:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:01.547739 13660 thread.cc:635] OpenStack (cloud detector) Time spent creating pthread: real 2.140s user 0.704s sys 1.385s
03:08:01.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:01.548504 13687 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2139 milliseconds
03:08:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:01.550563 13688 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:01.550671 13660 thread.cc:601] OpenStack (cloud detector) Time spent starting thread: real 2.143s user 0.705s sys 1.385s
03:08:01.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:01.551020 13660 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:08:01.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:01.561031 13660 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:01.574350 13660 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:01.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:01.578246 13660 hybrid_clock.cc:648] HybridClock initialized: now 1736824081578158 us; error 84 us; skew 500 ppm
03:08:01.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:01.579432 13660 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:01.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:08:01 dist-test-slave-rqxq krb5kdc[12807](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824081, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:01.589375 13660 init.cc:377] Logged in from keytab as kudu/127.12.126.189@KRBTEST.COM (short username kudu)
03:08:01.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:08:01 dist-test-slave-rqxq krb5kdc[12807](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824081, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.129@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:01.591984 13143 init.cc:295] Successfully reacquired a new kerberos TGT
03:08:01.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:01.621495 13660 webserver.cc:458] Webserver started at http://127.12.126.189:38527/ using document root <none> and password file <none>
03:08:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:01.622963 13660 fs_manager.cc:362] Metadata directory not provided
03:08:01.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:01.623297 13660 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:01.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:01.640995 13660 fs_manager.cc:714] Time spent opening directory manager: real 0.013s user 0.005s sys 0.006s
03:08:01.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:01.652673 13717 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:01.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:01.654070 13660 fs_manager.cc:730] Time spent opening block manager: real 0.010s user 0.001s sys 0.007s
03:08:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:01.654523 13660 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/data,/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/wal
03:08:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea2f415b454a4cee864155fdd4bbb343"
03:08:01.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:44 on dist-test-slave-rqxq"
03:08:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:01.658275 13660 fs_report.cc:389] FS layout report
03:08:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/wal
03:08:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/wal
03:08:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/data/data
03:08:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:01.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:01.796286 13660 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:01.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:01.809257 13660 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:01.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:01.809877 13660 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:02.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.277367 13772 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.126.189:35635 every 8 connection(s)
03:08:02.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.277971 13660 rpc_server.cc:307] RPC server started. Bound to: 127.12.126.189:35635
03:08:02.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.282603 13660 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-1/data/info.pb
03:08:02.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.290170 12794 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu as pid 13660
03:08:02.292 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:02.292 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.126.188:40957
03:08:02.292 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
03:08:02.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.294377 12794 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:08:02.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu
03:08:02.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/wal
03:08:02.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/data
03:08:02.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/logs
03:08:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/data/info.pb
03:08:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:02.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.126.188:40957
03:08:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.126.188
03:08:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35901
03:08:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.188.keytab
03:08:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.126.188
03:08:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:08:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:08:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.126.148:44839
03:08:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957
03:08:02.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34021/jwks.json
03:08:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:08:02.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.337611 13778 raft_consensus.cc:491] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:08:02.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.338114 13778 raft_consensus.cc:513] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } }
03:08:02.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.349720 13778 leader_election.cc:290] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers ea2f415b454a4cee864155fdd4bbb343 (127.12.126.189:35635), 16e48fc0bf4746988b005121cc02d88c (127.12.126.188:40957)
03:08:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:02.391446 13607 leader_election.cc:336] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 16e48fc0bf4746988b005121cc02d88c (127.12.126.188:40957): Network error: Client connection negotiation failed: client connection to 127.12.126.188:40957: connect: Connection refused (error 111)
03:08:02.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:08:02 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824078, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.190@KRBTEST.COM for kudu/127.12.126.189@KRBTEST.COM
03:08:02.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.392493 13773 sys_catalog.cc:263] Verifying existing consensus state
03:08:02.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:08:02 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824081, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.129@KRBTEST.COM for kudu/127.12.126.189@KRBTEST.COM
03:08:02.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.441917 13262 heartbeater.cc:346] Connected to a master server at 127.12.126.189:35635
03:08:02.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.445333 13773 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343: Bootstrap starting.
03:08:02.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.564000 13773 log.cc:826] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343: Log is configured to *not* fsync() on all Append() calls
03:08:02.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.650951 13773 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:08:02.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.654685 13773 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343: Bootstrap complete.
03:08:02.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.727897 13773 raft_consensus.cc:357] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } }
03:08:02.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.733724 13773 raft_consensus.cc:738] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: ea2f415b454a4cee864155fdd4bbb343, State: Initialized, Role: FOLLOWER
03:08:02.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.740774 13773 consensus_queue.cc:260] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } }
03:08:02.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.749480 13409 heartbeater.cc:346] Connected to a master server at 127.12.126.189:35635
03:08:02.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.778445 13748 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "ea2f415b454a4cee864155fdd4bbb343" is_pre_election: true
03:08:02.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.779460 13748 raft_consensus.cc:2463] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b32c0d9df04f4be0b9d7658cf31ee9d4 in term 1.
03:08:02.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.780997 13606 leader_election.cc:304] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b32c0d9df04f4be0b9d7658cf31ee9d4, ea2f415b454a4cee864155fdd4bbb343; no voters: 16e48fc0bf4746988b005121cc02d88c
03:08:02.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.774219 13789 sys_catalog.cc:455] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } } }
03:08:02.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.782507 13789 sys_catalog.cc:458] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [sys.catalog]: This master's current role is: FOLLOWER
03:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.783396 13778 raft_consensus.cc:2798] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [term 1 FOLLOWER]: Leader pre-election won for term 2
03:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.783776 13778 raft_consensus.cc:491] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:08:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.784127 13778 raft_consensus.cc:3054] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [term 1 FOLLOWER]: Advancing to term 2
03:08:02.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.796213 13773 sys_catalog.cc:564] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [sys.catalog]: configured and running, proceeding with master startup.
03:08:02.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.795768 13778 raft_consensus.cc:513] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } }
03:08:02.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.803239 13748 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "ea2f415b454a4cee864155fdd4bbb343"
03:08:02.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.803855 13748 raft_consensus.cc:3054] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [term 1 FOLLOWER]: Advancing to term 2
03:08:02.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.814792 13748 raft_consensus.cc:2463] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b32c0d9df04f4be0b9d7658cf31ee9d4 in term 2.
03:08:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.820607 13778 leader_election.cc:290] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [CANDIDATE]: Term 2 election: Requested vote from peers ea2f415b454a4cee864155fdd4bbb343 (127.12.126.189:35635), 16e48fc0bf4746988b005121cc02d88c (127.12.126.188:40957)
03:08:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.822283 13606 leader_election.cc:304] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: b32c0d9df04f4be0b9d7658cf31ee9d4, ea2f415b454a4cee864155fdd4bbb343; no voters:
03:08:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:02.822873 13607 leader_election.cc:336] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 16e48fc0bf4746988b005121cc02d88c (127.12.126.188:40957): Network error: Client connection negotiation failed: client connection to 127.12.126.188:40957: connect: Connection refused (error 111)
03:08:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.823495 13778 raft_consensus.cc:2798] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [term 2 FOLLOWER]: Leader election won for term 2
03:08:02.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.831882 13778 raft_consensus.cc:695] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [term 2 LEADER]: Becoming Leader. State: Replica: b32c0d9df04f4be0b9d7658cf31ee9d4, State: Running, Role: LEADER
03:08:02.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.833268 13778 consensus_queue.cc:237] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } }
03:08:02.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.865142 13791 sys_catalog.cc:455] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b32c0d9df04f4be0b9d7658cf31ee9d4. Latest consensus state: current_term: 2 leader_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } } }
03:08:02.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.866350 13791 sys_catalog.cc:458] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [sys.catalog]: This master's current role is: LEADER
03:08:02.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.883373 13800 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
03:08:02.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.906752 13800 catalog_manager.cc:1485] Initializing Kudu cluster ID...
03:08:02.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.911484 13800 catalog_manager.cc:1260] Loaded cluster ID: fc0b2ecc8c564017acad54664a84693a
03:08:02.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.911865 13800 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
03:08:02.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.915459 13803 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
03:08:02.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.937489 13800 catalog_manager.cc:1505] Loading token signing keys...
03:08:02.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:08:02 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824081, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.189@KRBTEST.COM for kudu/127.12.126.190@KRBTEST.COM
03:08:02.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:02.984107 13800 catalog_manager.cc:5910] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4: Loaded TSK: 0
03:08:03.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.005223 13800 catalog_manager.cc:1515] Initializing in-progress tserver states...
03:08:03.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:08:03 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824081, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.189@KRBTEST.COM for kudu/127.12.126.189@KRBTEST.COM
03:08:03.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.086051 13558 heartbeater.cc:346] Connected to a master server at 127.12.126.189:35635
03:08:03.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.138378 13738 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "dcb2a708806e4ce0a45e62f438922aa5" instance_seqno: 1736824076698615) as {username='kudu', principal='kudu/127.12.126.131@KRBTEST.COM'} at 127.12.126.131:54457; Asking this server to re-register.
03:08:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.144644 13558 heartbeater.cc:463] Registering TS with master...
03:08:03.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.070451 13660 master_runner.cc:186] Error getting master registration for 127.12.126.188:40957: Network error: Client connection negotiation failed: client connection to 127.12.126.188:40957: connect: Connection refused (error 111),
03:08:03.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.148020 13660 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:03.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.148819 13737 ts_manager.cc:194] Registered new tserver with Master: dcb2a708806e4ce0a45e62f438922aa5 (127.12.126.131:38289)
03:08:03.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:03.171025 13776 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:03.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:03.171869 13776 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:03.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:03.173060 13776 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:03.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:03.230868 13776 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:08:03.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:03.231173 13776 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:03.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:03.231673 13776 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:03.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:03.232023 13776 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:03.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:03.232380 13776 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:03.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:03.232712 13776 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:03.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:03.232973 13776 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34021/jwks.json
03:08:03.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:03.252295 13607 consensus_peers.cc:487] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 -> Peer 16e48fc0bf4746988b005121cc02d88c (127.12.126.188:40957): Couldn't send request to peer 16e48fc0bf4746988b005121cc02d88c. Status: Network error: Client connection negotiation failed: client connection to 127.12.126.188:40957: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:08:03.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.279441 13776 master_runner.cc:386] Master server non-default flags:
03:08:03.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:03.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.126.148:44839
03:08:03.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:03.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:03.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/data
03:08:03.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/wal
03:08:03.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:08:03.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:03.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957
03:08:03.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/krb5kdc/kudu_127.12.126.188.keytab
03:08:03.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.126.188
03:08:03.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:03.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:03.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:03.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.126.188:40957
03:08:03.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:03.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:03.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:03.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34021/jwks.json
03:08:03.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:03.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/data/info.pb
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.126.188
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35901
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/logs
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 02:43:23 UTC on a633016ff5c9
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4168
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.281153 13776 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.282471 13776 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:03.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:03.301638 13815 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:03.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.348685 13748 raft_consensus.cc:1270] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [term 2 FOLLOWER]: Refusing update from remote peer b32c0d9df04f4be0b9d7658cf31ee9d4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
03:08:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.351265 13791 consensus_queue.cc:1035] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
03:08:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.387364 13820 sys_catalog.cc:455] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b32c0d9df04f4be0b9d7658cf31ee9d4. Latest consensus state: current_term: 2 leader_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } } }
03:08:03.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.388474 13820 sys_catalog.cc:458] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [sys.catalog]: This master's current role is: FOLLOWER
03:08:03.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.410773 13820 sys_catalog.cc:455] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } } }
03:08:03.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.411936 13820 sys_catalog.cc:458] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343 [sys.catalog]: This master's current role is: FOLLOWER
03:08:03.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.416380 13791 sys_catalog.cc:455] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } } }
03:08:03.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.418016 13791 sys_catalog.cc:458] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [sys.catalog]: This master's current role is: LEADER
03:08:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.449024 13737 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "85d38da4c73c440db25182354e328654" instance_seqno: 1736824071986688) as {username='kudu', principal='kudu/127.12.126.129@KRBTEST.COM'} at 127.12.126.129:40899; Asking this server to re-register.
03:08:03.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.451349 13262 heartbeater.cc:463] Registering TS with master...
03:08:03.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.455386 13737 ts_manager.cc:194] Registered new tserver with Master: 85d38da4c73c440db25182354e328654 (127.12.126.129:43981)
03:08:03.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.755983 13737 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "5aa517d245cb498d8bc4c31b9f847d9e" instance_seqno: 1736824074444003) as {username='kudu', principal='kudu/127.12.126.130@KRBTEST.COM'} at 127.12.126.130:36849; Asking this server to re-register.
03:08:03.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.757859 13409 heartbeater.cc:463] Registering TS with master...
03:08:03.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.761906 13737 ts_manager.cc:194] Registered new tserver with Master: 5aa517d245cb498d8bc4c31b9f847d9e (127.12.126.130:34537)
03:08:03.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.940505 13803 catalog_manager.cc:1260] Loaded cluster ID: fc0b2ecc8c564017acad54664a84693a
03:08:03.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.940837 13803 catalog_manager.cc:1553] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343: loading cluster ID for follower catalog manager: success
03:08:03.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.958473 13803 catalog_manager.cc:1575] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343: acquiring CA information for follower catalog manager: success
03:08:03.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:03.977109 13803 catalog_manager.cc:1603] T 00000000000000000000000000000000 P ea2f415b454a4cee864155fdd4bbb343: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:08:04.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:04.114028 13575 master_runner.cc:186] Error getting master registration for 127.12.126.188:40957: Network error: Client connection negotiation failed: client connection to 127.12.126.188:40957: connect: Connection refused (error 111),
03:08:04.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:04.114575 13575 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:04.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:08:04 dist-test-slave-rqxq krb5kdc[12807](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824084, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.130@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:04.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:04.230310 13292 init.cc:295] Successfully reacquired a new kerberos TGT
03:08:04.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:04.396096 13267 heartbeater.cc:502] Master 127.12.126.190:35933 was elected leader, sending a full tablet report...
03:08:04.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:04.722671 13412 heartbeater.cc:502] Master 127.12.126.190:35933 was elected leader, sending a full tablet report...
03:08:04.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:04.933082 13560 heartbeater.cc:502] Master 127.12.126.190:35933 was elected leader, sending a full tablet report...
03:08:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:04.700526 13813 debug-util.cc:398] Leaking SignalData structure 0x7b080001cac0 after lost signal to thread 13776
03:08:05.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:05.084331 13816 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1779 milliseconds
03:08:05.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:05.082727 13776 thread.cc:635] OpenStack (cloud detector) Time spent creating pthread: real 1.779s user 0.544s sys 0.836s
03:08:05.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:05.084903 13776 thread.cc:601] OpenStack (cloud detector) Time spent starting thread: real 1.782s user 0.545s sys 0.836s
03:08:05.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:05.085889 13817 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:05.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:03.301909 13814 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:05.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.086763 13776 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:08:05.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.090611 13776 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:05.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.094482 13776 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:05.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.095922 13776 hybrid_clock.cc:648] HybridClock initialized: now 1736824085095884 us; error 40 us; skew 500 ppm
03:08:05.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.096706 13776 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:05.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:08:05 dist-test-slave-rqxq krb5kdc[12807](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824085, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:05.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.102519 13776 init.cc:377] Logged in from keytab as kudu/127.12.126.188@KRBTEST.COM (short username kudu)
03:08:05.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.108742 13776 webserver.cc:458] Webserver started at http://127.12.126.188:35901/ using document root <none> and password file <none>
03:08:05.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.109683 13776 fs_manager.cc:362] Metadata directory not provided
03:08:05.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.109911 13776 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:05.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.118253 13776 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.005s sys 0.001s
03:08:05.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.123359 13841 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:05.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.124454 13776 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
03:08:05.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.124774 13776 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/data,/tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/wal
03:08:05.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "16e48fc0bf4746988b005121cc02d88c"
03:08:05.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 03:07:46 on dist-test-slave-rqxq"
03:08:05.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.127095 13776 fs_report.cc:389] FS layout report
03:08:05.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:05.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/wal
03:08:05.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/wal
03:08:05.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/data/data
03:08:05.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:05.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:05.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:05.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:05.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:05.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:05.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:05.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:05.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:05.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:05.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:05.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.203091 13776 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:05.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.211411 13776 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:05.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.212260 13776 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.360577 13776 rpc_server.cc:307] RPC server started. Bound to: 127.12.126.188:40957
03:08:05.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.361865 13892 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.126.188:40957 every 8 connection(s)
03:08:05.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.365463 13776 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNdHK51/test-tmp/mini-kudu-cluster4093739672702096469/master-2/data/info.pb
03:08:05.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.374444 12794 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu as pid 13776
03:08:05.375 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:05.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.377552 13893 sys_catalog.cc:263] Verifying existing consensus state
03:08:05.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.384459 13893 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c: Bootstrap starting.
03:08:05.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:08:05 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824081, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.129@KRBTEST.COM for kudu/127.12.126.188@KRBTEST.COM
03:08:05.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.444166 13893 log.cc:826] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c: Log is configured to *not* fsync() on all Append() calls
03:08:05.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.460795 13263 heartbeater.cc:346] Connected to a master server at 127.12.126.188:40957
03:08:05.469 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.126.190:35933(127.12.126.190:35933)] connecting to peer
03:08:05.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.487238 13893 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:08:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.488543 13893 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c: Bootstrap complete.
03:08:05.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.521808 13893 raft_consensus.cc:357] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } }
03:08:05.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.525411 13893 raft_consensus.cc:738] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 16e48fc0bf4746988b005121cc02d88c, State: Initialized, Role: FOLLOWER
03:08:05.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.526458 13893 consensus_queue.cc:260] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } }
03:08:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.555150 13893 sys_catalog.cc:564] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [sys.catalog]: configured and running, proceeding with master startup.
03:08:05.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.555887 13900 sys_catalog.cc:455] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } } }
03:08:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.556974 13900 sys_catalog.cc:458] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [sys.catalog]: This master's current role is: FOLLOWER
03:08:05.576 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.126.189:35635(127.12.126.189:35635)] connecting to peer
03:08:05.580 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.126.188:40957(127.12.126.188:40957)] connecting to peer
03:08:05.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:08:05 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824078, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.190@KRBTEST.COM for kudu/127.12.126.188@KRBTEST.COM
03:08:05.648 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.12.126.189:35635(127.12.126.189:35635)] Successfully connected to peer
03:08:05.651 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.12.126.190:35933(127.12.126.190:35933)] Successfully connected to peer
03:08:05.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.658491 13868 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [term 1 FOLLOWER]: Advancing to term 2
03:08:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.670989 13913 catalog_manager.cc:1260] Loaded cluster ID: fc0b2ecc8c564017acad54664a84693a
03:08:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.671272 13913 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c: loading cluster ID for follower catalog manager: success
03:08:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.678519 13913 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c: acquiring CA information for follower catalog manager: success
03:08:05.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 03:08:05 dist-test-slave-rqxq krb5kdc[12807](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736824085, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.126.188@KRBTEST.COM for kudu/127.12.126.190@KRBTEST.COM
03:08:05.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.704136 13913 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:08:05.786 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.12.126.188:40957(127.12.126.188:40957)] Successfully connected to peer
03:08:05.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.787081 13922 sys_catalog.cc:455] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } } }
03:08:05.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.788049 13922 sys_catalog.cc:458] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [sys.catalog]: This master's current role is: FOLLOWER
03:08:05.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.789348 13900 sys_catalog.cc:455] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [sys.catalog]: SysCatalogTable state changed. Reason: New leader b32c0d9df04f4be0b9d7658cf31ee9d4. Latest consensus state: current_term: 2 leader_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } } }
03:08:05.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.790030 13900 sys_catalog.cc:458] T 00000000000000000000000000000000 P 16e48fc0bf4746988b005121cc02d88c [sys.catalog]: This master's current role is: FOLLOWER
03:08:05.810 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
03:08:05.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.816147 13410 heartbeater.cc:346] Connected to a master server at 127.12.126.188:40957
03:08:05.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.824226 13858 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "5aa517d245cb498d8bc4c31b9f847d9e" instance_seqno: 1736824074444003) as {username='kudu', principal='kudu/127.12.126.130@KRBTEST.COM'} at 127.12.126.130:53669; Asking this server to re-register.
03:08:05.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.825757 13410 heartbeater.cc:463] Registering TS with master...
03:08:05.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:05.829540 13857 ts_manager.cc:194] Registered new tserver with Master: 5aa517d245cb498d8bc4c31b9f847d9e (127.12.126.130:34537)
03:08:05.832 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.12.126.189
03:08:05.833 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.12.126.190
03:08:05.999 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.12.126.188
03:08:06.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:06.105235 13823 sys_catalog.cc:455] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b32c0d9df04f4be0b9d7658cf31ee9d4" member_type: VOTER last_known_addr { host: "127.12.126.190" port: 35933 } } peers { permanent_uuid: "ea2f415b454a4cee864155fdd4bbb343" member_type: VOTER last_known_addr { host: "127.12.126.189" port: 35635 } } peers { permanent_uuid: "16e48fc0bf4746988b005121cc02d88c" member_type: VOTER last_known_addr { host: "127.12.126.188" port: 40957 } } }
03:08:06.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:06.106019 13823 sys_catalog.cc:458] T 00000000000000000000000000000000 P b32c0d9df04f4be0b9d7658cf31ee9d4 [sys.catalog]: This master's current role is: LEADER
03:08:06.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:06.114964 13559 heartbeater.cc:346] Connected to a master server at 127.12.126.188:40957
03:08:06.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:06.115480 13559 heartbeater.cc:502] Master 127.12.126.188:40957 was elected leader, sending a full tablet report...
03:08:06.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:06.119218 13858 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "dcb2a708806e4ce0a45e62f438922aa5" instance_seqno: 1736824076698615) as {username='kudu', principal='kudu/127.12.126.131@KRBTEST.COM'} at 127.12.126.131:55311; Asking this server to re-register.
03:08:06.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:06.124055 13559 heartbeater.cc:463] Registering TS with master...
03:08:06.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:06.126910 13858 ts_manager.cc:194] Registered new tserver with Master: dcb2a708806e4ce0a45e62f438922aa5 (127.12.126.131:38289)
03:08:06.147 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.126.189:35635(127.12.126.189:35635)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
03:08:06.148 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.126.190:35933(127.12.126.190:35933)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
03:08:06.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.126.190:35933: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
03:08:06.153 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.126.189:35635: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
03:08:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:06.154404 13917 negotiation.cc:337] Failed RPC negotiation. Trace:
03:08:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.640694 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:49146 (local address 127.12.126.190:35933)
03:08:06.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.669405 (+ 28711us) server_negotiation.cc:206] Beginning negotiation
03:08:06.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.669433 (+ 28us) server_negotiation.cc:399] Waiting for connection header
03:08:06.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.744172 (+ 74739us) server_negotiation.cc:407] Connection header received
03:08:06.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.785525 (+ 41353us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
03:08:06.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.785554 (+ 29us) server_negotiation.cc:461] Received NEGOTIATE request from client
03:08:06.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.785789 (+ 235us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
03:08:06.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.785939 (+ 150us) server_negotiation.cc:226] Negotiated authn=SASL
03:08:06.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.992295 (+206356us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
03:08:06.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.994101 (+ 1806us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
03:08:06.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:06.147238 (+153137us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
03:08:06.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:06.147888 (+ 650us) server_negotiation.cc:662] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
03:08:06.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:06.152191 (+ 4303us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:49146: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:49146): Cannot send after transport endpoint shutdown (error 108)
03:08:06.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":28081,"thread_start_us":28670,"threads_started":1}
03:08:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:06.157464 13916 negotiation.cc:337] Failed RPC negotiation. Trace:
03:08:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.639853 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:60110 (local address 127.12.126.189:35635)
03:08:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.660665 (+ 20812us) server_negotiation.cc:206] Beginning negotiation
03:08:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.660690 (+ 25us) server_negotiation.cc:399] Waiting for connection header
03:08:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.735213 (+ 74523us) server_negotiation.cc:407] Connection header received
03:08:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.779738 (+ 44525us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
03:08:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.779767 (+ 29us) server_negotiation.cc:461] Received NEGOTIATE request from client
03:08:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.779988 (+ 221us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
03:08:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.780723 (+ 735us) server_negotiation.cc:226] Negotiated authn=SASL
03:08:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.990623 (+209900us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
03:08:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.992625 (+ 2002us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
03:08:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:06.131740 (+139115us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
03:08:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:06.132512 (+ 772us) server_negotiation.cc:662] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
03:08:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:06.155674 (+ 23162us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:60110: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:60110): Cannot send after transport endpoint shutdown (error 108)
03:08:06.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":20189,"thread_start_us":20517,"threads_started":1}
03:08:06.161 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.12.126.189:35635(127.12.126.189:35635)] handling channelInactive
03:08:06.183 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.126.188:40957(127.12.126.188:40957)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
03:08:06.184 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.126.188:40957: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
03:08:06.189 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=809), Traces: [0ms] refreshing cache from master, [65ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.126.190:35933, [191ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.126.189:35635, [195ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.126.188:40957, [764ms] Sub RPC ConnectToMaster: received response from server master-127.12.126.190:35933: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [763ms] Sub RPC ConnectToMaster: received response from server master-127.12.126.189:35635: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [798ms] Sub RPC ConnectToMaster: received response from server master-127.12.126.188:40957: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) [main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) [main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) [async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) [main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) [main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) [main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) [netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
03:08:06.203 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=823), Traces: [0ms] refreshing cache from master, [65ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.126.190:35933, [191ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.126.189:35635, [195ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.126.188:40957, [764ms] Sub RPC ConnectToMaster: received response from server master-127.12.126.190:35933: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [763ms] Sub RPC ConnectToMaster: received response from server master-127.12.126.189:35635: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [798ms] Sub RPC ConnectToMaster: received response from server master-127.12.126.188:40957: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) with no deferred attached!
03:08:06.206 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.12.126.188:40957(127.12.126.188:40957)] disconnecting while in state TERMINATED
03:08:06.207 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.12.126.190:35933(127.12.126.190:35933)] disconnecting while in state TERMINATED
03:08:06.207 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.12.126.189:35635(127.12.126.189:35635)] disconnecting while in state TERMINATED
03:08:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:06.215195 13897 negotiation.cc:337] Failed RPC negotiation. Trace:
03:08:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.650492 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:43066 (local address 127.12.126.188:40957)
03:08:06.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.651451 (+ 959us) server_negotiation.cc:206] Beginning negotiation
03:08:06.216 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.12.126.190:35933(127.12.126.190:35933)] handling channelInactive
03:08:06.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.651485 (+ 34us) server_negotiation.cc:399] Waiting for connection header
03:08:06.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.787093 (+135608us) server_negotiation.cc:407] Connection header received
03:08:06.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.787882 (+ 789us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
03:08:06.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.787911 (+ 29us) server_negotiation.cc:461] Received NEGOTIATE request from client
03:08:06.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.788141 (+ 230us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
03:08:06.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:05.788290 (+ 149us) server_negotiation.cc:226] Negotiated authn=SASL
03:08:06.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:06.044755 (+256465us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
03:08:06.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:06.046415 (+ 1660us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
03:08:06.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:06.181704 (+135289us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
03:08:06.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:06.182328 (+ 624us) server_negotiation.cc:662] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
03:08:06.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 03:08:06.213428 (+ 31100us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:43066: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:43066): Cannot send after transport endpoint shutdown (error 108)
03:08:06.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":143}
03:08:06.218 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:08:06.234 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.12.126.188:40957(127.12.126.188:40957)] handling channelInactive
03:08:06.236 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:08:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:06.248235 12794 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu with pid 13106
03:08:06.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:06.295519 12794 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu with pid 13274
03:08:06.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:06.330060 12794 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu with pid 13421
03:08:06.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:06.345188 12295 debug-util.cc:398] Leaking SignalData structure 0x7b080009f7a0 after lost signal to thread 12227
03:08:06.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 03:08:06.345875 12295 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba4e0 after lost signal to thread 12298
03:08:06.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:06.380796 12794 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu with pid 13575
03:08:06.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:06.447203 12794 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu with pid 13660
03:08:06.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:06.481732 12794 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu with pid 13776
03:08:07.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-14T03:08:07Z chronyd exiting
03:08:08.731 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:08:08.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:08.732518 12065 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu with pid 12347
03:08:08.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:08.756448 12065 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu with pid 12510
03:08:08.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:08.782832 12065 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu with pid 12653
03:08:08.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:08.808121 12065 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu with pid 12077
03:08:08.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:08.835813 12065 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu with pid 12147
03:08:08.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 03:08:08.864620 12065 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskNdHK51/build/tsan/bin/kudu with pid 12222
03:08:09.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-14T03:08:09Z chronyd exiting
03:08:11.042 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
03:08:12.480 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
03:08:12.480 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.12.126.190:35933,127.12.126.189:35635,127.12.126.188:40957). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more