14:05:47.222 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
14:05:47.234 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskqLh0aI/test-tmp
14:05:47.237 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:05:47.240 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskqLh0aI/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:05:47.556 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
14:05:47.679 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:05:47.681 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:05:47.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:47.684384 24065 mini_oidc.cc:150] Starting JWKS server
14:05:47.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:47.687940 24065 webserver.cc:480] Webserver started at http://127.0.0.1:43105/ using document root <none> and password file <none>
14:05:47.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:47.688370 24065 mini_oidc.cc:186] Starting OIDC Discovery server
14:05:47.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:47.690037 24065 webserver.cc:480] Webserver started at http://127.0.0.1:36585/ using document root <none> and password file <none>
14:05:47.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-08-31T14:05:47Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:05:47.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-08-31T14:05:47Z Disabled control of system clock
14:05:47.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:47.749944 24065 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:05:47.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:05:47.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-0/wal
14:05:47.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-0/data
14:05:47.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:05:47.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:05:47.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:05:47.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:47.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:47.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:47.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-0/logs
14:05:47.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-0/data/info.pb
14:05:47.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:47.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:05:47.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:05:47.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:05:47.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:05:47.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:47.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:47.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:47.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:05:47.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:05:47.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:05:47.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:05:47.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.128.126:42075
14:05:47.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.128.126
14:05:47.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:47.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.128.84:42325
14:05:47.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:47.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:47.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:05:47.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:05:47.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.128.126:42075,127.23.128.125:36771,127.23.128.124:45273
14:05:47.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:05:47.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43105/jwks.json with env {}
14:05:48.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:48.046768 24077 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:05:48.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:48.047330 24077 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:05:48.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:48.047890 24077 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:05:48.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:48.079196 24077 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:05:48.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:48.079489 24077 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:05:48.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:48.079680 24077 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:05:48.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:48.079905 24077 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:05:48.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:48.080173 24077 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:05:48.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:48.080317 24077 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43105/jwks.json
14:05:48.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:48.115677 24077 master_runner.cc:386] Master server non-default flags:
14:05:48.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:48.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.128.84:42325
14:05:48.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:48.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:05:48.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-0/data
14:05:48.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-0/wal
14:05:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:05:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.128.126:42075,127.23.128.125:36771,127.23.128.124:45273
14:05:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:05:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.128.126:42075
14:05:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:05:48.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:05:48.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:05:48.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43105/jwks.json
14:05:48.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:48.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:48.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-0/data/info.pb
14:05:48.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.128.126
14:05:48.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:48.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:05:48.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:48.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:05:48.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:05:48.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:48.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-0/logs
14:05:48.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:48.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:05:48.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:05:48.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:05:48.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:05:48.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:05:48.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:05:48.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 13:43:23 UTC on 5fd53c4cbb9d
14:05:48.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7833
14:05:48.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:05:48.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:48.117251 24077 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:05:48.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:48.119484 24077 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:05:48.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:48.132097 24083 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:49.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:48.132560 24084 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:49.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:49.469409 24086 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:49.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:49.470050 24085 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1337 milliseconds
14:05:49.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:49.472775 24077 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.341s user 0.476s sys 0.854s
14:05:49.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:49.473145 24077 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.342s user 0.476s sys 0.854s
14:05:49.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.473457 24077 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:05:49.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.476421 24077 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:05:49.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.479871 24077 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:05:49.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.481670 24077 hybrid_clock.cc:648] HybridClock initialized: now 1756649149481630 us; error 36 us; skew 500 ppm
14:05:49.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.482951 24077 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:05:49.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.492372 24077 webserver.cc:480] Webserver started at http://127.23.128.126:41489/ using document root <none> and password file <none>
14:05:49.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.493824 24077 fs_manager.cc:362] Metadata directory not provided
14:05:49.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.494102 24077 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:05:49.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.494758 24077 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:05:49.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.502257 24077 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-0/data/instance:
14:05:49.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "48433a692a1d4149a877e7f7d68fceaf"
14:05:49.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:05:49 on dist-test-slave-sg9n"
14:05:49.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.504035 24077 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-0/wal/instance:
14:05:49.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "48433a692a1d4149a877e7f7d68fceaf"
14:05:49.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:05:49 on dist-test-slave-sg9n"
14:05:49.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.514622 24077 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.003s sys 0.008s
14:05:49.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.522985 24093 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:05:49.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.524117 24077 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
14:05:49.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.524417 24077 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-0/data,/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-0/wal
14:05:49.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "48433a692a1d4149a877e7f7d68fceaf"
14:05:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:05:49 on dist-test-slave-sg9n"
14:05:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.524730 24077 fs_report.cc:389] FS layout report
14:05:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:05:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-0/wal
14:05:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-0/wal
14:05:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-0/data/data
14:05:49.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:05:49.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:05:49.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:05:49.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:05:49.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:05:49.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:05:49.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:05:49.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:05:49.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:05:49.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:05:49.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:05:49.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.603775 24077 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:05:49.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.605216 24077 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:05:49.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.605621 24077 kserver.cc:163] Server-wide thread pool size limit: 3276
14:05:49.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.678061 24077 rpc_server.cc:307] RPC server started. Bound to: 127.23.128.126:42075
14:05:49.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.678114 24144 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.128.126:42075 every 8 connection(s)
14:05:49.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.680768 24077 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-0/data/info.pb
14:05:49.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.681962 24065 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu as pid 24077
14:05:49.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.682407 24065 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-0/wal/instance
14:05:49.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.687181 24145 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:05:49.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.687150 24065 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:05:49.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:05:49.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-1/wal
14:05:49.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-1/data
14:05:49.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:05:49.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:05:49.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:05:49.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:49.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:49.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:49.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-1/logs
14:05:49.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-1/data/info.pb
14:05:49.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:49.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:05:49.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:05:49.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:05:49.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:05:49.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:49.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:49.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:49.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:05:49.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:05:49.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:05:49.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:05:49.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.128.125:36771
14:05:49.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.128.125
14:05:49.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:49.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.128.84:42325
14:05:49.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:49.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:49.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:05:49.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:05:49.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.128.126:42075,127.23.128.125:36771,127.23.128.124:45273
14:05:49.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:05:49.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43105/jwks.json with env {}
14:05:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.702721 24145 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.128.126" port: 42075 } has no permanent_uuid. Determining permanent_uuid...
14:05:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.730605 24145 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } has no permanent_uuid. Determining permanent_uuid...
14:05:49.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:49.733122 24097 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.128.125:36771: connect: Connection refused (error 111)
14:05:49.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:49.737576 24145 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.23.128.125:36771: Network error: Client connection negotiation failed: client connection to 127.23.128.125:36771: connect: Connection refused (error 111)
14:05:49.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.787176 24145 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } attempt: 1
14:05:49.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:49.791873 24145 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.23.128.125:36771: Network error: Client connection negotiation failed: client connection to 127.23.128.125:36771: connect: Connection refused (error 111)
14:05:49.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.860360 24145 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } attempt: 2
14:05:49.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:49.864727 24145 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.23.128.125:36771: Network error: Client connection negotiation failed: client connection to 127.23.128.125:36771: connect: Connection refused (error 111)
14:05:49.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:49.956126 24145 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } attempt: 3
14:05:49.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:49.960013 24145 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.23.128.125:36771: Network error: Client connection negotiation failed: client connection to 127.23.128.125:36771: connect: Connection refused (error 111)
14:05:50.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:50.004356 24147 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:05:50.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:50.004928 24147 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:05:50.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:50.005434 24147 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:05:50.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:50.036747 24147 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:05:50.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:50.037047 24147 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:05:50.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:50.037245 24147 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:05:50.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:50.037429 24147 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:05:50.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:50.037616 24147 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:05:50.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:50.037773 24147 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43105/jwks.json
14:05:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:50.073426 24147 master_runner.cc:386] Master server non-default flags:
14:05:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.128.84:42325
14:05:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:05:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-1/data
14:05:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-1/wal
14:05:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:05:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.128.126:42075,127.23.128.125:36771,127.23.128.124:45273
14:05:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:05:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.128.125:36771
14:05:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:05:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:05:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:05:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43105/jwks.json
14:05:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-1/data/info.pb
14:05:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.128.125
14:05:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:05:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:05:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:05:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-1/logs
14:05:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:05:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:05:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:05:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:05:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:05:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:05:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 13:43:23 UTC on 5fd53c4cbb9d
14:05:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7833
14:05:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:05:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:50.074759 24147 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:05:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:50.077047 24147 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:05:50.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:50.089246 24156 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:50.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:50.103487 24145 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } attempt: 4
14:05:50.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:50.110067 24145 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.23.128.125:36771: Network error: Client connection negotiation failed: client connection to 127.23.128.125:36771: connect: Connection refused (error 111)
14:05:50.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:50.409690 24145 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } attempt: 5
14:05:50.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:50.418926 24145 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.23.128.125:36771: Network error: Client connection negotiation failed: client connection to 127.23.128.125:36771: connect: Connection refused (error 111)
14:05:50.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:50.966583 24145 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } attempt: 6
14:05:50.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:50.991065 24145 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.23.128.125:36771: Network error: Client connection negotiation failed: client connection to 127.23.128.125:36771: connect: Connection refused (error 111)
14:05:51.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:51.494904 24155 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ec0 after lost signal to thread 24147
14:05:51.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:51.498134 24155 kernel_stack_watchdog.cc:198] Thread 24147 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 402ms:
14:05:51.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:05:51.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:05:51.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:05:51.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:05:51.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:51.498791 24147 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.409s user 0.483s sys 0.925s
14:05:51.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:51.499243 24147 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.410s user 0.483s sys 0.925s
14:05:51.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:50.089730 24157 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:51.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:51.500682 24158 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1409 milliseconds
14:05:51.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:51.501525 24159 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:51.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.502099 24147 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:05:51.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.507066 24147 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:05:51.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.509471 24147 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:05:51.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.510790 24147 hybrid_clock.cc:648] HybridClock initialized: now 1756649151510763 us; error 46 us; skew 500 ppm
14:05:51.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.511525 24147 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:05:51.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.517549 24147 webserver.cc:480] Webserver started at http://127.23.128.125:35835/ using document root <none> and password file <none>
14:05:51.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.518402 24147 fs_manager.cc:362] Metadata directory not provided
14:05:51.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.518585 24147 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:05:51.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.519029 24147 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:05:51.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.523389 24147 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-1/data/instance:
14:05:51.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a9b5d61b5789479887b708325ba1ce6d"
14:05:51.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:05:51 on dist-test-slave-sg9n"
14:05:51.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.524475 24147 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-1/wal/instance:
14:05:51.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a9b5d61b5789479887b708325ba1ce6d"
14:05:51.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:05:51 on dist-test-slave-sg9n"
14:05:51.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.533339 24147 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.005s sys 0.005s
14:05:51.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.540323 24168 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:05:51.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.541456 24147 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.002s
14:05:51.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.541751 24147 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-1/data,/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-1/wal
14:05:51.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a9b5d61b5789479887b708325ba1ce6d"
14:05:51.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:05:51 on dist-test-slave-sg9n"
14:05:51.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.542061 24147 fs_report.cc:389] FS layout report
14:05:51.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:05:51.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-1/wal
14:05:51.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-1/wal
14:05:51.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-1/data/data
14:05:51.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:05:51.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:05:51.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:05:51.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:05:51.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:05:51.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:05:51.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:05:51.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:05:51.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:05:51.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:05:51.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:05:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.589419 24147 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:05:51.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.590884 24147 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:05:51.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.591303 24147 kserver.cc:163] Server-wide thread pool size limit: 3276
14:05:51.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.663615 24147 rpc_server.cc:307] RPC server started. Bound to: 127.23.128.125:36771
14:05:51.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.663676 24219 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.128.125:36771 every 8 connection(s)
14:05:51.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.666150 24147 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-1/data/info.pb
14:05:51.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.669900 24065 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu as pid 24147
14:05:51.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.670362 24065 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-1/wal/instance
14:05:51.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.671726 24220 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:05:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.673295 24065 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:05:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:05:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-2/wal
14:05:51.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-2/data
14:05:51.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:05:51.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:05:51.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:05:51.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:51.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:51.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:51.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-2/logs
14:05:51.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-2/data/info.pb
14:05:51.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:51.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:05:51.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:05:51.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:05:51.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:05:51.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:51.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:51.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:51.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:05:51.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:05:51.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:05:51.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:05:51.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.128.124:45273
14:05:51.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.128.124
14:05:51.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:51.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.128.84:42325
14:05:51.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:51.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:51.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:05:51.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:05:51.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.128.126:42075,127.23.128.125:36771,127.23.128.124:45273
14:05:51.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:05:51.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43105/jwks.json with env {}
14:05:51.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.685624 24220 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.128.126" port: 42075 } has no permanent_uuid. Determining permanent_uuid...
14:05:51.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.713368 24220 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } has no permanent_uuid. Determining permanent_uuid...
14:05:51.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.729146 24220 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } has no permanent_uuid. Determining permanent_uuid...
14:05:51.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:51.731598 24170 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.128.124:45273: connect: Connection refused (error 111)
14:05:51.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:51.735769 24220 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.23.128.124:45273: Network error: Client connection negotiation failed: client connection to 127.23.128.124:45273: connect: Connection refused (error 111)
14:05:51.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.785231 24220 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } attempt: 1
14:05:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:51.789549 24220 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.23.128.124:45273: Network error: Client connection negotiation failed: client connection to 127.23.128.124:45273: connect: Connection refused (error 111)
14:05:51.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.857981 24220 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } attempt: 2
14:05:51.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:51.862380 24220 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.23.128.124:45273: Network error: Client connection negotiation failed: client connection to 127.23.128.124:45273: connect: Connection refused (error 111)
14:05:51.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:51.953775 24220 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } attempt: 3
14:05:51.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:51.959018 24220 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.23.128.124:45273: Network error: Client connection negotiation failed: client connection to 127.23.128.124:45273: connect: Connection refused (error 111)
14:05:51.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:51.993366 24222 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:05:51.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:51.993999 24222 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:05:51.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:51.994473 24222 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:05:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:52.025775 24222 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:05:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:52.026060 24222 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:05:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:52.026257 24222 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:05:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:52.026446 24222 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:05:52.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:52.026777 24222 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:05:52.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:52.027033 24222 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43105/jwks.json
14:05:52.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:52.051594 24145 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } attempt: 7
14:05:52.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:52.063416 24145 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } has no permanent_uuid. Determining permanent_uuid...
14:05:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:52.067615 24222 master_runner.cc:386] Master server non-default flags:
14:05:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.128.84:42325
14:05:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:05:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-2/data
14:05:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-2/wal
14:05:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:05:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.128.126:42075,127.23.128.125:36771,127.23.128.124:45273
14:05:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:05:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.128.124:45273
14:05:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:05:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:05:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:05:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43105/jwks.json
14:05:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-2/data/info.pb
14:05:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.128.124
14:05:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:05:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:05:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:05:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-2/logs
14:05:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:52.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:05:52.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:05:52.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:05:52.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:05:52.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:05:52.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:05:52.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 13:43:23 UTC on 5fd53c4cbb9d
14:05:52.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7833
14:05:52.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:05:52.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:52.068729 24145 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.23.128.124:45273: Network error: Client connection negotiation failed: client connection to 127.23.128.124:45273: connect: Connection refused (error 111)
14:05:52.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:52.069290 24222 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:05:52.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:52.071579 24222 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:05:52.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:52.083302 24234 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:52.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:52.102523 24220 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } attempt: 4
14:05:52.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:52.110105 24220 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.23.128.124:45273: Network error: Client connection negotiation failed: client connection to 127.23.128.124:45273: connect: Connection refused (error 111)
14:05:52.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:52.127102 24145 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } attempt: 1
14:05:52.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:52.132308 24145 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.23.128.124:45273: Network error: Client connection negotiation failed: client connection to 127.23.128.124:45273: connect: Connection refused (error 111)
14:05:52.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:52.213842 24145 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } attempt: 2
14:05:52.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:52.221326 24145 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.23.128.124:45273: Network error: Client connection negotiation failed: client connection to 127.23.128.124:45273: connect: Connection refused (error 111)
14:05:52.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:52.306874 24145 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } attempt: 3
14:05:52.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:52.315814 24145 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.23.128.124:45273: Network error: Client connection negotiation failed: client connection to 127.23.128.124:45273: connect: Connection refused (error 111)
14:05:52.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:52.409667 24220 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } attempt: 5
14:05:52.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:52.417115 24220 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.23.128.124:45273: Network error: Client connection negotiation failed: client connection to 127.23.128.124:45273: connect: Connection refused (error 111)
14:05:52.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:52.456499 24145 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } attempt: 4
14:05:52.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:52.464222 24145 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.23.128.124:45273: Network error: Client connection negotiation failed: client connection to 127.23.128.124:45273: connect: Connection refused (error 111)
14:05:52.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:52.747766 24145 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } attempt: 5
14:05:52.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:52.755430 24145 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.23.128.124:45273: Network error: Client connection negotiation failed: client connection to 127.23.128.124:45273: connect: Connection refused (error 111)
14:05:52.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:52.964673 24220 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } attempt: 6
14:05:53.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:53.006850 24220 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.23.128.124:45273: Network error: Client connection negotiation failed: client connection to 127.23.128.124:45273: connect: Connection refused (error 111)
14:05:53.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.308007 24145 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } attempt: 6
14:05:53.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:53.339668 24145 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.23.128.124:45273: Network error: Client connection negotiation failed: client connection to 127.23.128.124:45273: connect: Connection refused (error 111)
14:05:53.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:53.487018 24233 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ec0 after lost signal to thread 24222
14:05:53.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:53.567298 24222 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.484s user 0.465s sys 1.003s
14:05:53.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:53.567730 24222 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.484s user 0.465s sys 1.003s
14:05:53.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:52.087670 24235 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:53.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:53.568907 24236 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1485 milliseconds
14:05:53.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:53.569664 24237 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:53.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.569597 24222 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:05:53.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.572793 24222 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:05:53.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.575244 24222 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:05:53.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.576591 24222 hybrid_clock.cc:648] HybridClock initialized: now 1756649153576547 us; error 44 us; skew 500 ppm
14:05:53.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.577450 24222 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:05:53.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.586814 24222 webserver.cc:480] Webserver started at http://127.23.128.124:39947/ using document root <none> and password file <none>
14:05:53.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.587863 24222 fs_manager.cc:362] Metadata directory not provided
14:05:53.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.588104 24222 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:05:53.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.588611 24222 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:05:53.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.593168 24222 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-2/data/instance:
14:05:53.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "46d8575172234b57913af56e733dc1be"
14:05:53.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:05:53 on dist-test-slave-sg9n"
14:05:53.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.594425 24222 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-2/wal/instance:
14:05:53.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "46d8575172234b57913af56e733dc1be"
14:05:53.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:05:53 on dist-test-slave-sg9n"
14:05:53.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.606499 24222 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.005s sys 0.005s
14:05:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.614931 24248 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:05:53.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.616190 24222 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
14:05:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.617000 24222 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-2/data,/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-2/wal
14:05:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "46d8575172234b57913af56e733dc1be"
14:05:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:05:53 on dist-test-slave-sg9n"
14:05:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.617940 24222 fs_report.cc:389] FS layout report
14:05:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:05:53.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-2/wal
14:05:53.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-2/wal
14:05:53.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-2/data/data
14:05:53.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:05:53.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:05:53.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:05:53.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:05:53.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:05:53.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:05:53.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:05:53.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:05:53.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:05:53.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:05:53.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:05:53.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.677834 24222 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:05:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.679657 24222 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:05:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.680261 24222 kserver.cc:163] Server-wide thread pool size limit: 3276
14:05:53.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.753679 24222 rpc_server.cc:307] RPC server started. Bound to: 127.23.128.124:45273
14:05:53.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.753737 24299 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.128.124:45273 every 8 connection(s)
14:05:53.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.756597 24222 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-2/data/info.pb
14:05:53.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.761953 24300 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:05:53.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.763382 24065 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu as pid 24222
14:05:53.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.763804 24065 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/master-2/wal/instance
14:05:53.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.781648 24300 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.128.126" port: 42075 } has no permanent_uuid. Determining permanent_uuid...
14:05:53.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.806236 24300 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } has no permanent_uuid. Determining permanent_uuid...
14:05:53.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.816888 24300 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } has no permanent_uuid. Determining permanent_uuid...
14:05:53.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.834218 24300 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be: Bootstrap starting.
14:05:53.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.841511 24300 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be: Neither blocks nor log segments found. Creating new log.
14:05:53.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.843472 24300 log.cc:826] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be: Log is configured to *not* fsync() on all Append() calls
14:05:53.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.847996 24300 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be: No bootstrap required, opened a new log
14:05:53.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.867290 24300 raft_consensus.cc:357] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "48433a692a1d4149a877e7f7d68fceaf" member_type: VOTER last_known_addr { host: "127.23.128.126" port: 42075 } } peers { permanent_uuid: "a9b5d61b5789479887b708325ba1ce6d" member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } } peers { permanent_uuid: "46d8575172234b57913af56e733dc1be" member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } }
14:05:53.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.868033 24300 raft_consensus.cc:383] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:05:53.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.868306 24300 raft_consensus.cc:738] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 46d8575172234b57913af56e733dc1be, State: Initialized, Role: FOLLOWER
14:05:53.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.869143 24300 consensus_queue.cc:260] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "48433a692a1d4149a877e7f7d68fceaf" member_type: VOTER last_known_addr { host: "127.23.128.126" port: 42075 } } peers { permanent_uuid: "a9b5d61b5789479887b708325ba1ce6d" member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } } peers { permanent_uuid: "46d8575172234b57913af56e733dc1be" member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } }
14:05:53.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.871776 24309 sys_catalog.cc:455] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "48433a692a1d4149a877e7f7d68fceaf" member_type: VOTER last_known_addr { host: "127.23.128.126" port: 42075 } } peers { permanent_uuid: "a9b5d61b5789479887b708325ba1ce6d" member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } } peers { permanent_uuid: "46d8575172234b57913af56e733dc1be" member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } } }
14:05:53.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.872823 24309 sys_catalog.cc:458] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [sys.catalog]: This master's current role is: FOLLOWER
14:05:53.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.873862 24300 sys_catalog.cc:564] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [sys.catalog]: configured and running, proceeding with master startup.
14:05:53.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:53.899129 24320 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:05:53.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:53.899515 24320 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:05:53.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.894385 24222 master_runner.cc:186] Error getting master registration for 127.23.128.126:42075: OK, instance_id { permanent_uuid: "48433a692a1d4149a877e7f7d68fceaf" instance_seqno: 1756649149658130 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:05:53.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:53.915856 24222 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:05:53.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:53.935765 24254 tablet.cc:2378] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:05:54.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.067440 24220 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } attempt: 7
14:05:54.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.087881 24220 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d: Bootstrap starting.
14:05:54.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.095508 24220 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d: Neither blocks nor log segments found. Creating new log.
14:05:54.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.097882 24220 log.cc:826] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d: Log is configured to *not* fsync() on all Append() calls
14:05:54.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.103206 24220 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d: No bootstrap required, opened a new log
14:05:54.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.124799 24220 raft_consensus.cc:357] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "48433a692a1d4149a877e7f7d68fceaf" member_type: VOTER last_known_addr { host: "127.23.128.126" port: 42075 } } peers { permanent_uuid: "a9b5d61b5789479887b708325ba1ce6d" member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } } peers { permanent_uuid: "46d8575172234b57913af56e733dc1be" member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } }
14:05:54.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.125520 24220 raft_consensus.cc:383] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:05:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.125809 24220 raft_consensus.cc:738] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a9b5d61b5789479887b708325ba1ce6d, State: Initialized, Role: FOLLOWER
14:05:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.126783 24220 consensus_queue.cc:260] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "48433a692a1d4149a877e7f7d68fceaf" member_type: VOTER last_known_addr { host: "127.23.128.126" port: 42075 } } peers { permanent_uuid: "a9b5d61b5789479887b708325ba1ce6d" member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } } peers { permanent_uuid: "46d8575172234b57913af56e733dc1be" member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } }
14:05:54.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.129524 24324 sys_catalog.cc:455] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "48433a692a1d4149a877e7f7d68fceaf" member_type: VOTER last_known_addr { host: "127.23.128.126" port: 42075 } } peers { permanent_uuid: "a9b5d61b5789479887b708325ba1ce6d" member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } } peers { permanent_uuid: "46d8575172234b57913af56e733dc1be" member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } } }
14:05:54.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.130527 24324 sys_catalog.cc:458] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d [sys.catalog]: This master's current role is: FOLLOWER
14:05:54.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.131510 24220 sys_catalog.cc:564] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d [sys.catalog]: configured and running, proceeding with master startup.
14:05:54.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:54.173508 24335 catalog_manager.cc:1560] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:05:54.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:54.175014 24335 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:05:54.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.159072 24147 master_runner.cc:186] Error getting master registration for 127.23.128.126:42075: OK, instance_id { permanent_uuid: "48433a692a1d4149a877e7f7d68fceaf" instance_seqno: 1756649149658130 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:05:54.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.191545 24147 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:05:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:54.349562 24174 tablet.cc:2378] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:05:54.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.367261 24309 raft_consensus.cc:491] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:05:54.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.367830 24309 raft_consensus.cc:513] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "48433a692a1d4149a877e7f7d68fceaf" member_type: VOTER last_known_addr { host: "127.23.128.126" port: 42075 } } peers { permanent_uuid: "a9b5d61b5789479887b708325ba1ce6d" member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } } peers { permanent_uuid: "46d8575172234b57913af56e733dc1be" member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } }
14:05:54.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.370370 24309 leader_election.cc:290] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 48433a692a1d4149a877e7f7d68fceaf (127.23.128.126:42075), a9b5d61b5789479887b708325ba1ce6d (127.23.128.125:36771)
14:05:54.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.372231 24195 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "46d8575172234b57913af56e733dc1be" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a9b5d61b5789479887b708325ba1ce6d" is_pre_election: true
14:05:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.373081 24195 raft_consensus.cc:2466] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 46d8575172234b57913af56e733dc1be in term 0.
14:05:54.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.374004 24145 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } attempt: 7
14:05:54.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.374538 24252 leader_election.cc:304] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 46d8575172234b57913af56e733dc1be, a9b5d61b5789479887b708325ba1ce6d; no voters:
14:05:54.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.382556 24309 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [term 0 FOLLOWER]: Leader pre-election won for term 1
14:05:54.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.382951 24309 raft_consensus.cc:491] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:05:54.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.383270 24309 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [term 0 FOLLOWER]: Advancing to term 1
14:05:54.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.390597 24309 raft_consensus.cc:513] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "48433a692a1d4149a877e7f7d68fceaf" member_type: VOTER last_known_addr { host: "127.23.128.126" port: 42075 } } peers { permanent_uuid: "a9b5d61b5789479887b708325ba1ce6d" member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } } peers { permanent_uuid: "46d8575172234b57913af56e733dc1be" member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } }
14:05:54.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.396090 24195 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "46d8575172234b57913af56e733dc1be" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a9b5d61b5789479887b708325ba1ce6d"
14:05:54.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.396654 24195 raft_consensus.cc:3058] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d [term 0 FOLLOWER]: Advancing to term 1
14:05:54.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.404176 24195 raft_consensus.cc:2466] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 46d8575172234b57913af56e733dc1be in term 1.
14:05:54.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.412521 24252 leader_election.cc:304] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 46d8575172234b57913af56e733dc1be, a9b5d61b5789479887b708325ba1ce6d; no voters:
14:05:54.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.414575 24309 leader_election.cc:290] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [CANDIDATE]: Term 1 election: Requested vote from peers 48433a692a1d4149a877e7f7d68fceaf (127.23.128.126:42075), a9b5d61b5789479887b708325ba1ce6d (127.23.128.125:36771)
14:05:54.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.415608 24309 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [term 1 FOLLOWER]: Leader election won for term 1
14:05:54.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.416826 24309 raft_consensus.cc:695] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [term 1 LEADER]: Becoming Leader. State: Replica: 46d8575172234b57913af56e733dc1be, State: Running, Role: LEADER
14:05:54.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.418421 24309 consensus_queue.cc:237] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "48433a692a1d4149a877e7f7d68fceaf" member_type: VOTER last_known_addr { host: "127.23.128.126" port: 42075 } } peers { permanent_uuid: "a9b5d61b5789479887b708325ba1ce6d" member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } } peers { permanent_uuid: "46d8575172234b57913af56e733dc1be" member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } }
14:05:54.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.439895 24145 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 48433a692a1d4149a877e7f7d68fceaf: Bootstrap starting.
14:05:54.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.459399 24338 sys_catalog.cc:455] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [sys.catalog]: SysCatalogTable state changed. Reason: New leader 46d8575172234b57913af56e733dc1be. Latest consensus state: current_term: 1 leader_uuid: "46d8575172234b57913af56e733dc1be" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "48433a692a1d4149a877e7f7d68fceaf" member_type: VOTER last_known_addr { host: "127.23.128.126" port: 42075 } } peers { permanent_uuid: "a9b5d61b5789479887b708325ba1ce6d" member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } } peers { permanent_uuid: "46d8575172234b57913af56e733dc1be" member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } } }
14:05:54.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.460186 24338 sys_catalog.cc:458] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [sys.catalog]: This master's current role is: LEADER
14:05:54.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.471624 24340 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:05:54.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.474516 24145 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 48433a692a1d4149a877e7f7d68fceaf: Neither blocks nor log segments found. Creating new log.
14:05:54.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.479952 24340 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:05:54.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.480226 24145 log.cc:826] T 00000000000000000000000000000000 P 48433a692a1d4149a877e7f7d68fceaf: Log is configured to *not* fsync() on all Append() calls
14:05:54.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.504895 24195 raft_consensus.cc:1273] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d [term 1 FOLLOWER]: Refusing update from remote peer 46d8575172234b57913af56e733dc1be: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:05:54.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.507485 24338 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [LEADER]: Connected to new peer: Peer: permanent_uuid: "a9b5d61b5789479887b708325ba1ce6d" member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
14:05:54.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.508616 24145 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 48433a692a1d4149a877e7f7d68fceaf: No bootstrap required, opened a new log
14:05:54.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.541023 24324 sys_catalog.cc:455] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 46d8575172234b57913af56e733dc1be. Latest consensus state: current_term: 1 leader_uuid: "46d8575172234b57913af56e733dc1be" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "48433a692a1d4149a877e7f7d68fceaf" member_type: VOTER last_known_addr { host: "127.23.128.126" port: 42075 } } peers { permanent_uuid: "a9b5d61b5789479887b708325ba1ce6d" member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } } peers { permanent_uuid: "46d8575172234b57913af56e733dc1be" member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } } }
14:05:54.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.541761 24324 sys_catalog.cc:458] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d [sys.catalog]: This master's current role is: FOLLOWER
14:05:54.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.548899 24342 mvcc.cc:204] Tried to move back new op lower bound from 7195234936836206592 to 7195234936513961984. Current Snapshot: MvccSnapshot[applied={T|T < 7195234936836206592}]
14:05:54.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.555684 24309 sys_catalog.cc:455] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "46d8575172234b57913af56e733dc1be" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "48433a692a1d4149a877e7f7d68fceaf" member_type: VOTER last_known_addr { host: "127.23.128.126" port: 42075 } } peers { permanent_uuid: "a9b5d61b5789479887b708325ba1ce6d" member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } } peers { permanent_uuid: "46d8575172234b57913af56e733dc1be" member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } } }
14:05:54.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.556412 24309 sys_catalog.cc:458] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [sys.catalog]: This master's current role is: LEADER
14:05:54.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.558197 24340 catalog_manager.cc:1349] Generated new cluster ID: 2e543356744242ff8ce290ee968ea7e1
14:05:54.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.558517 24340 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:05:54.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.557674 24343 mvcc.cc:204] Tried to move back new op lower bound from 7195234936836206592 to 7195234936513961984. Current Snapshot: MvccSnapshot[applied={T|T < 7195234936836206592}]
14:05:54.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.569211 24324 sys_catalog.cc:455] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "46d8575172234b57913af56e733dc1be" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "48433a692a1d4149a877e7f7d68fceaf" member_type: VOTER last_known_addr { host: "127.23.128.126" port: 42075 } } peers { permanent_uuid: "a9b5d61b5789479887b708325ba1ce6d" member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } } peers { permanent_uuid: "46d8575172234b57913af56e733dc1be" member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } } }
14:05:54.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.569866 24324 sys_catalog.cc:458] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d [sys.catalog]: This master's current role is: FOLLOWER
14:05:54.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.610432 24145 raft_consensus.cc:357] T 00000000000000000000000000000000 P 48433a692a1d4149a877e7f7d68fceaf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "48433a692a1d4149a877e7f7d68fceaf" member_type: VOTER last_known_addr { host: "127.23.128.126" port: 42075 } } peers { permanent_uuid: "a9b5d61b5789479887b708325ba1ce6d" member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } } peers { permanent_uuid: "46d8575172234b57913af56e733dc1be" member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } }
14:05:54.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.613374 24145 raft_consensus.cc:383] T 00000000000000000000000000000000 P 48433a692a1d4149a877e7f7d68fceaf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:05:54.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.615056 24145 raft_consensus.cc:738] T 00000000000000000000000000000000 P 48433a692a1d4149a877e7f7d68fceaf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 48433a692a1d4149a877e7f7d68fceaf, State: Initialized, Role: FOLLOWER
14:05:54.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.618994 24340 catalog_manager.cc:1372] Generated new certificate authority record
14:05:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.618105 24145 consensus_queue.cc:260] T 00000000000000000000000000000000 P 48433a692a1d4149a877e7f7d68fceaf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "48433a692a1d4149a877e7f7d68fceaf" member_type: VOTER last_known_addr { host: "127.23.128.126" port: 42075 } } peers { permanent_uuid: "a9b5d61b5789479887b708325ba1ce6d" member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } } peers { permanent_uuid: "46d8575172234b57913af56e733dc1be" member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } }
14:05:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.621618 24340 catalog_manager.cc:1506] Loading token signing keys...
14:05:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.653331 24118 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 48433a692a1d4149a877e7f7d68fceaf [term 0 FOLLOWER]: Advancing to term 1
14:05:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.654934 24119 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "46d8575172234b57913af56e733dc1be" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "48433a692a1d4149a877e7f7d68fceaf"
14:05:54.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.660194 24120 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "46d8575172234b57913af56e733dc1be" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "48433a692a1d4149a877e7f7d68fceaf" is_pre_election: true
14:05:54.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.660583 24340 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be: Generated new TSK 0
14:05:54.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.661669 24340 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:05:54.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.678171 24065 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:05:54.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:05:54.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-0/wal
14:05:54.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-0/data
14:05:54.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:05:54.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:05:54.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:05:54.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:54.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:54.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:54.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-0/logs
14:05:54.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-0/data/info.pb
14:05:54.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:54.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:05:54.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:05:54.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:05:54.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:05:54.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:54.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:54.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:54.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:05:54.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:05:54.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.128.65:0
14:05:54.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.128.65
14:05:54.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.128.65
14:05:54.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:54.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.128.126:42075,127.23.128.125:36771,127.23.128.124:45273
14:05:54.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.128.84:42325
14:05:54.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:54.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:54.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:05:54.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.685838 24118 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 48433a692a1d4149a877e7f7d68fceaf [term 1 FOLLOWER]: Refusing update from remote peer 46d8575172234b57913af56e733dc1be: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:05:54.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.651398 24347 sys_catalog.cc:455] T 00000000000000000000000000000000 P 48433a692a1d4149a877e7f7d68fceaf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "48433a692a1d4149a877e7f7d68fceaf" member_type: VOTER last_known_addr { host: "127.23.128.126" port: 42075 } } peers { permanent_uuid: "a9b5d61b5789479887b708325ba1ce6d" member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } } peers { permanent_uuid: "46d8575172234b57913af56e733dc1be" member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } } }
14:05:54.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.691349 24145 sys_catalog.cc:564] T 00000000000000000000000000000000 P 48433a692a1d4149a877e7f7d68fceaf [sys.catalog]: configured and running, proceeding with master startup.
14:05:54.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.691551 24347 sys_catalog.cc:458] T 00000000000000000000000000000000 P 48433a692a1d4149a877e7f7d68fceaf [sys.catalog]: This master's current role is: FOLLOWER
14:05:54.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.691433 24338 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [LEADER]: Connected to new peer: Peer: permanent_uuid: "48433a692a1d4149a877e7f7d68fceaf" member_type: VOTER last_known_addr { host: "127.23.128.126" port: 42075 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:05:54.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.753021 24347 sys_catalog.cc:455] T 00000000000000000000000000000000 P 48433a692a1d4149a877e7f7d68fceaf [sys.catalog]: SysCatalogTable state changed. Reason: New leader 46d8575172234b57913af56e733dc1be. Latest consensus state: current_term: 1 leader_uuid: "46d8575172234b57913af56e733dc1be" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "48433a692a1d4149a877e7f7d68fceaf" member_type: VOTER last_known_addr { host: "127.23.128.126" port: 42075 } } peers { permanent_uuid: "a9b5d61b5789479887b708325ba1ce6d" member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } } peers { permanent_uuid: "46d8575172234b57913af56e733dc1be" member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } } }
14:05:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.753243 24353 sys_catalog.cc:455] T 00000000000000000000000000000000 P 48433a692a1d4149a877e7f7d68fceaf [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "46d8575172234b57913af56e733dc1be" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "48433a692a1d4149a877e7f7d68fceaf" member_type: VOTER last_known_addr { host: "127.23.128.126" port: 42075 } } peers { permanent_uuid: "a9b5d61b5789479887b708325ba1ce6d" member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } } peers { permanent_uuid: "46d8575172234b57913af56e733dc1be" member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } } }
14:05:54.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.754139 24353 sys_catalog.cc:458] T 00000000000000000000000000000000 P 48433a692a1d4149a877e7f7d68fceaf [sys.catalog]: This master's current role is: FOLLOWER
14:05:54.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.754101 24347 sys_catalog.cc:458] T 00000000000000000000000000000000 P 48433a692a1d4149a877e7f7d68fceaf [sys.catalog]: This master's current role is: FOLLOWER
14:05:54.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:54.837868 24365 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:05:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:55.127167 24338 sys_catalog.cc:455] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "46d8575172234b57913af56e733dc1be" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "48433a692a1d4149a877e7f7d68fceaf" member_type: VOTER last_known_addr { host: "127.23.128.126" port: 42075 } } peers { permanent_uuid: "a9b5d61b5789479887b708325ba1ce6d" member_type: VOTER last_known_addr { host: "127.23.128.125" port: 36771 } } peers { permanent_uuid: "46d8575172234b57913af56e733dc1be" member_type: VOTER last_known_addr { host: "127.23.128.124" port: 45273 } } }
14:05:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:55.127907 24338 sys_catalog.cc:458] T 00000000000000000000000000000000 P 46d8575172234b57913af56e733dc1be [sys.catalog]: This master's current role is: LEADER
14:05:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:55.181730 24335 catalog_manager.cc:1261] Loaded cluster ID: 2e543356744242ff8ce290ee968ea7e1
14:05:55.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:55.182533 24335 catalog_manager.cc:1554] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d: loading cluster ID for follower catalog manager: success
14:05:55.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:55.190126 24335 catalog_manager.cc:1576] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d: acquiring CA information for follower catalog manager: success
14:05:55.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:55.197839 24335 catalog_manager.cc:1604] T 00000000000000000000000000000000 P a9b5d61b5789479887b708325ba1ce6d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:05:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:55.259503 24348 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:05:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:55.260218 24348 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:05:55.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:55.261015 24348 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:05:55.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:55.319334 24348 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:05:55.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:55.320772 24348 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.128.65
14:05:55.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:55.384748 24348 tablet_server_runner.cc:78] Tablet server non-default flags:
14:05:55.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:55.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.128.84:42325
14:05:55.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:55.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:05:55.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-0/data
14:05:55.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-0/wal
14:05:55.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:55.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:55.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.128.65:0
14:05:55.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:05:55.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:55.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:55.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-0/data/info.pb
14:05:55.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.128.65
14:05:55.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:55.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.128.126:42075,127.23.128.125:36771,127.23.128.124:45273
14:05:55.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:05:55.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:55.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:05:55.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:05:55.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:55.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.128.65
14:05:55.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-0/logs
14:05:55.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:05:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:05:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:05:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:05:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:05:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:05:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 13:43:23 UTC on 5fd53c4cbb9d
14:05:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7833
14:05:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:05:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:55.386724 24348 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:05:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:55.388957 24348 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:05:55.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:55.405128 24371 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:55.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:55.867055 24365 catalog_manager.cc:1261] Loaded cluster ID: 2e543356744242ff8ce290ee968ea7e1
14:05:55.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:55.874624 24365 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 48433a692a1d4149a877e7f7d68fceaf: loading cluster ID for follower catalog manager: success
14:05:55.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:55.882335 24365 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 48433a692a1d4149a877e7f7d68fceaf: acquiring CA information for follower catalog manager: success
14:05:55.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:55.916205 24365 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 48433a692a1d4149a877e7f7d68fceaf: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:05:57.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:56.805732 24370 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 24348
14:05:57.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:57.167953 24348 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.761s user 0.617s sys 1.008s
14:05:57.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:55.419399 24372 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:57.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:57.196732 24348 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.790s user 0.617s sys 1.008s
14:05:57.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.197110 24348 server_base.cc:1047] running on GCE node
14:05:57.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:57.197811 24374 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:57.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.199324 24348 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:05:57.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.202100 24348 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:05:57.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.203532 24348 hybrid_clock.cc:648] HybridClock initialized: now 1756649157203422 us; error 112 us; skew 500 ppm
14:05:57.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.204612 24348 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:05:57.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.212497 24348 webserver.cc:480] Webserver started at http://127.23.128.65:40873/ using document root <none> and password file <none>
14:05:57.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.213778 24348 fs_manager.cc:362] Metadata directory not provided
14:05:57.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.214067 24348 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:05:57.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.214596 24348 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:05:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.221055 24348 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-0/data/instance:
14:05:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a205031cf69f41efa0cdf56f6bbe60f6"
14:05:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:05:57 on dist-test-slave-sg9n"
14:05:57.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.222550 24348 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-0/wal/instance:
14:05:57.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a205031cf69f41efa0cdf56f6bbe60f6"
14:05:57.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:05:57 on dist-test-slave-sg9n"
14:05:57.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.232120 24348 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.006s sys 0.002s
14:05:57.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.239666 24382 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:05:57.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.241324 24348 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
14:05:57.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.241729 24348 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-0/data,/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-0/wal
14:05:57.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a205031cf69f41efa0cdf56f6bbe60f6"
14:05:57.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:05:57 on dist-test-slave-sg9n"
14:05:57.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.242165 24348 fs_report.cc:389] FS layout report
14:05:57.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:05:57.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-0/wal
14:05:57.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-0/wal
14:05:57.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-0/data/data
14:05:57.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:05:57.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:05:57.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:05:57.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:05:57.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:05:57.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:05:57.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:05:57.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:05:57.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:05:57.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:05:57.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:05:57.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.336077 24348 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:05:57.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.338045 24348 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:05:57.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.338572 24348 kserver.cc:163] Server-wide thread pool size limit: 3276
14:05:57.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.343520 24348 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:05:57.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.349393 24348 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:05:57.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.349691 24348 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:05:57.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.349987 24348 ts_tablet_manager.cc:610] Registered 0 tablets
14:05:57.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.350203 24348 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:05:57.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.532431 24348 rpc_server.cc:307] RPC server started. Bound to: 127.23.128.65:40287
14:05:57.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.532562 24494 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.128.65:40287 every 8 connection(s)
14:05:57.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.536019 24348 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-0/data/info.pb
14:05:57.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.547608 24065 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu as pid 24348
14:05:57.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.548076 24065 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-0/wal/instance
14:05:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.581254 24065 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:05:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:05:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-1/wal
14:05:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-1/data
14:05:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:05:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:05:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:05:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-1/logs
14:05:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-1/data/info.pb
14:05:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:05:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:05:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:05:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:05:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:57.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:57.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:57.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:05:57.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:05:57.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.128.66:0
14:05:57.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.128.66
14:05:57.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.128.66
14:05:57.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:57.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.128.126:42075,127.23.128.125:36771,127.23.128.124:45273
14:05:57.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.128.84:42325
14:05:57.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:57.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:57.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:05:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.616050 24496 heartbeater.cc:344] Connected to a master server at 127.23.128.125:36771
14:05:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.616550 24496 heartbeater.cc:461] Registering TS with master...
14:05:57.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.617784 24496 heartbeater.cc:507] Master 127.23.128.125:36771 requested a full tablet report, sending...
14:05:57.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.621238 24185 ts_manager.cc:194] Registered new tserver with Master: a205031cf69f41efa0cdf56f6bbe60f6 (127.23.128.65:40287)
14:05:57.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.624599 24501 heartbeater.cc:344] Connected to a master server at 127.23.128.126:42075
14:05:57.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.624959 24501 heartbeater.cc:461] Registering TS with master...
14:05:57.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.625862 24501 heartbeater.cc:507] Master 127.23.128.126:42075 requested a full tablet report, sending...
14:05:57.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.629745 24110 ts_manager.cc:194] Registered new tserver with Master: a205031cf69f41efa0cdf56f6bbe60f6 (127.23.128.65:40287)
14:05:57.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.635779 24495 heartbeater.cc:344] Connected to a master server at 127.23.128.124:45273
14:05:57.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.636124 24495 heartbeater.cc:461] Registering TS with master...
14:05:57.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.637014 24495 heartbeater.cc:507] Master 127.23.128.124:45273 requested a full tablet report, sending...
14:05:57.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.640630 24265 ts_manager.cc:194] Registered new tserver with Master: a205031cf69f41efa0cdf56f6bbe60f6 (127.23.128.65:40287)
14:05:57.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:57.645264 24265 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.128.65:53327
14:05:58.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:58.000411 24505 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:05:58.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:58.000921 24505 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:05:58.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:58.001435 24505 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:05:58.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:58.035257 24505 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:05:58.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:58.036988 24505 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.128.66
14:05:58.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:58.077679 24505 tablet_server_runner.cc:78] Tablet server non-default flags:
14:05:58.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:05:58.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.128.84:42325
14:05:58.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:05:58.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:05:58.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-1/data
14:05:58.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-1/wal
14:05:58.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:05:58.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:05:58.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.128.66:0
14:05:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:05:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:05:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:05:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-1/data/info.pb
14:05:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.128.66
14:05:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:05:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.128.126:42075,127.23.128.125:36771,127.23.128.124:45273
14:05:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:05:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:05:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:05:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:05:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:05:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.128.66
14:05:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-1/logs
14:05:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:05:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:05:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:05:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:05:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:05:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:05:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:05:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 13:43:23 UTC on 5fd53c4cbb9d
14:05:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7833
14:05:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:05:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:58.079564 24505 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:05:58.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:58.081504 24505 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:05:58.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:58.095345 24513 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:58.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:58.649240 24495 heartbeater.cc:499] Master 127.23.128.124:45273 was elected leader, sending a full tablet report...
14:05:59.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:58.097308 24514 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:59.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:59.496461 24512 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 24505
14:05:59.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:59.964169 24505 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.869s user 0.617s sys 1.094s
14:05:59.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:59.964540 24505 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.869s user 0.617s sys 1.094s
14:05:59.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:59.966812 24516 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:05:59.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:05:59.968412 24515 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1868 milliseconds
14:05:59.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:59.968431 24505 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:05:59.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:59.969655 24505 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:05:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:59.971810 24505 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:05:59.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:59.973179 24505 hybrid_clock.cc:648] HybridClock initialized: now 1756649159973135 us; error 49 us; skew 500 ppm
14:05:59.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:59.974141 24505 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:05:59.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:59.980456 24505 webserver.cc:480] Webserver started at http://127.23.128.66:34241/ using document root <none> and password file <none>
14:05:59.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:59.981469 24505 fs_manager.cc:362] Metadata directory not provided
14:05:59.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:59.981715 24505 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:05:59.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:59.982306 24505 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:05:59.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:59.986851 24505 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-1/data/instance:
14:05:59.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "604e86591f2b41119b6355676b422b0d"
14:05:59.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:05:59 on dist-test-slave-sg9n"
14:05:59.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:59.988023 24505 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-1/wal/instance:
14:05:59.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "604e86591f2b41119b6355676b422b0d"
14:05:59.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:05:59 on dist-test-slave-sg9n"
14:05:59.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:05:59.995733 24505 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
14:06:00.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.004153 24524 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:06:00.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.005667 24505 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.001s sys 0.004s
14:06:00.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.006026 24505 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-1/data,/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-1/wal
14:06:00.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "604e86591f2b41119b6355676b422b0d"
14:06:00.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:05:59 on dist-test-slave-sg9n"
14:06:00.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.006382 24505 fs_report.cc:389] FS layout report
14:06:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:06:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-1/wal
14:06:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-1/wal
14:06:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-1/data/data
14:06:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:06:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:06:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:06:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:06:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:06:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:06:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:06:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:06:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:06:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:06:00.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:06:00.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.054386 24505 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:06:00.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.056133 24505 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:06:00.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.056787 24505 kserver.cc:163] Server-wide thread pool size limit: 3276
14:06:00.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.059710 24505 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:06:00.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.063984 24505 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:06:00.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.064203 24505 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:06:00.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.064508 24505 ts_tablet_manager.cc:610] Registered 0 tablets
14:06:00.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.064673 24505 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:06:00.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.203368 24505 rpc_server.cc:307] RPC server started. Bound to: 127.23.128.66:33049
14:06:00.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.203418 24636 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.128.66:33049 every 8 connection(s)
14:06:00.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.206260 24505 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-1/data/info.pb
14:06:00.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.213950 24065 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu as pid 24505
14:06:00.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.214771 24065 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-1/wal/instance
14:06:00.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.249589 24065 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:06:00.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:06:00.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-2/wal
14:06:00.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-2/data
14:06:00.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:06:00.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:06:00.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:06:00.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:00.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:00.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:00.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-2/logs
14:06:00.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-2/data/info.pb
14:06:00.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:00.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:06:00.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:06:00.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:06:00.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:06:00.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:00.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:00.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:00.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:06:00.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:06:00.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.128.67:0
14:06:00.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.128.67
14:06:00.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.128.67
14:06:00.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:00.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.128.126:42075,127.23.128.125:36771,127.23.128.124:45273
14:06:00.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.128.84:42325
14:06:00.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:00.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:00.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:06:00.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.254372 24638 heartbeater.cc:344] Connected to a master server at 127.23.128.125:36771
14:06:00.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.254954 24638 heartbeater.cc:461] Registering TS with master...
14:06:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.256280 24638 heartbeater.cc:507] Master 127.23.128.125:36771 requested a full tablet report, sending...
14:06:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.256350 24639 heartbeater.cc:344] Connected to a master server at 127.23.128.126:42075
14:06:00.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.256759 24639 heartbeater.cc:461] Registering TS with master...
14:06:00.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.257673 24639 heartbeater.cc:507] Master 127.23.128.126:42075 requested a full tablet report, sending...
14:06:00.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.259775 24185 ts_manager.cc:194] Registered new tserver with Master: 604e86591f2b41119b6355676b422b0d (127.23.128.66:33049)
14:06:00.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.260850 24110 ts_manager.cc:194] Registered new tserver with Master: 604e86591f2b41119b6355676b422b0d (127.23.128.66:33049)
14:06:00.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.261381 24637 heartbeater.cc:344] Connected to a master server at 127.23.128.124:45273
14:06:00.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.261708 24637 heartbeater.cc:461] Registering TS with master...
14:06:00.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.262657 24637 heartbeater.cc:507] Master 127.23.128.124:45273 requested a full tablet report, sending...
14:06:00.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.264943 24265 ts_manager.cc:194] Registered new tserver with Master: 604e86591f2b41119b6355676b422b0d (127.23.128.66:33049)
14:06:00.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.266721 24265 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.128.66:60477
14:06:00.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:00.556326 24647 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:06:00.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:00.556802 24647 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:06:00.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:00.557226 24647 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:06:00.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:00.589068 24647 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:06:00.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:00.589874 24647 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.128.67
14:06:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.626957 24647 tablet_server_runner.cc:78] Tablet server non-default flags:
14:06:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.128.84:42325
14:06:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-2/data
14:06:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-2/wal
14:06:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.128.67:0
14:06:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:06:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-2/data/info.pb
14:06:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.128.67
14:06:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.128.126:42075,127.23.128.125:36771,127.23.128.124:45273
14:06:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:06:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:06:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:06:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.128.67
14:06:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-2/logs
14:06:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:06:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:06:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:06:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:06:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:06:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:06:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 13:43:23 UTC on 5fd53c4cbb9d
14:06:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7833
14:06:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:06:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.628314 24647 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:06:00.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:00.629895 24647 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:06:00.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:00.641597 24653 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:01.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:01.270812 24637 heartbeater.cc:499] Master 127.23.128.124:45273 was elected leader, sending a full tablet report...
14:06:02.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:02.045753 24652 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 24647
14:06:02.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:02.165171 24652 kernel_stack_watchdog.cc:198] Thread 24647 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
14:06:02.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:06:02.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:06:02.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:06:02.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:06:02.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:02.165841 24647 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.523s user 0.426s sys 1.032s
14:06:02.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:00.643323 24654 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:02.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:02.166323 24647 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.524s user 0.426s sys 1.032s
14:06:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:02.166967 24655 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1522 milliseconds
14:06:02.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.167809 24647 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:06:02.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:02.167902 24656 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:02.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.171005 24647 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:06:02.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.173184 24647 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:06:02.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.174536 24647 hybrid_clock.cc:648] HybridClock initialized: now 1756649162174496 us; error 49 us; skew 500 ppm
14:06:02.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.175336 24647 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:06:02.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.181283 24647 webserver.cc:480] Webserver started at http://127.23.128.67:44163/ using document root <none> and password file <none>
14:06:02.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.182155 24647 fs_manager.cc:362] Metadata directory not provided
14:06:02.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.182334 24647 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:06:02.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.182735 24647 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:06:02.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.187080 24647 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-2/data/instance:
14:06:02.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "278ac53829b744e4949b3edd6dea19d3"
14:06:02.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:02 on dist-test-slave-sg9n"
14:06:02.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.188122 24647 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-2/wal/instance:
14:06:02.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "278ac53829b744e4949b3edd6dea19d3"
14:06:02.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:02 on dist-test-slave-sg9n"
14:06:02.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.195300 24647 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.002s
14:06:02.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.201234 24664 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:06:02.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.202399 24647 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
14:06:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.202724 24647 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-2/data,/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-2/wal
14:06:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "278ac53829b744e4949b3edd6dea19d3"
14:06:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:02 on dist-test-slave-sg9n"
14:06:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.203038 24647 fs_report.cc:389] FS layout report
14:06:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:06:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-2/wal
14:06:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-2/wal
14:06:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-2/data/data
14:06:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:06:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:06:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:06:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:06:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:06:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:06:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:06:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:06:02.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:06:02.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:06:02.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:06:02.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.250416 24647 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:06:02.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.251888 24647 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:06:02.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.252301 24647 kserver.cc:163] Server-wide thread pool size limit: 3276
14:06:02.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.255030 24647 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:06:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.259356 24647 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:06:02.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.259578 24647 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:06:02.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.259864 24647 ts_tablet_manager.cc:610] Registered 0 tablets
14:06:02.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.260036 24647 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:06:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.396322 24647 rpc_server.cc:307] RPC server started. Bound to: 127.23.128.67:33303
14:06:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.396430 24776 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.23.128.67:33303 every 8 connection(s)
14:06:02.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.398743 24647 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-2/data/info.pb
14:06:02.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.406183 24065 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu as pid 24647
14:06:02.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.406658 24065 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster228277409621020639/ts-2/wal/instance
14:06:02.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.437379 24778 heartbeater.cc:344] Connected to a master server at 127.23.128.125:36771
14:06:02.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.438028 24777 heartbeater.cc:344] Connected to a master server at 127.23.128.124:45273
14:06:02.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.438299 24779 heartbeater.cc:344] Connected to a master server at 127.23.128.126:42075
14:06:02.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.437912 24778 heartbeater.cc:461] Registering TS with master...
14:06:02.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.438805 24777 heartbeater.cc:461] Registering TS with master...
14:06:02.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.438855 24779 heartbeater.cc:461] Registering TS with master...
14:06:02.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.439972 24778 heartbeater.cc:507] Master 127.23.128.125:36771 requested a full tablet report, sending...
14:06:02.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.440018 24779 heartbeater.cc:507] Master 127.23.128.126:42075 requested a full tablet report, sending...
14:06:02.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.440066 24777 heartbeater.cc:507] Master 127.23.128.124:45273 requested a full tablet report, sending...
14:06:02.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.443225 24185 ts_manager.cc:194] Registered new tserver with Master: 278ac53829b744e4949b3edd6dea19d3 (127.23.128.67:33303)
14:06:02.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.443228 24110 ts_manager.cc:194] Registered new tserver with Master: 278ac53829b744e4949b3edd6dea19d3 (127.23.128.67:33303)
14:06:02.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.445010 24265 ts_manager.cc:194] Registered new tserver with Master: 278ac53829b744e4949b3edd6dea19d3 (127.23.128.67:33303)
14:06:02.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.446759 24265 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.128.67:51729
14:06:02.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:02.452962 24065 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
14:06:02.463 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:06:02.465 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
14:06:02.480 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.23.128.126"
port: 42075
}
bound_http_address {
host: "127.23.128.126"
port: 41489
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.23.128.125"
port: 36771
}
bound_http_address {
host: "127.23.128.125"
port: 35835
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.23.128.124"
port: 45273
}
bound_http_address {
host: "127.23.128.124"
port: 39947
}
}
}
14:06:02.484 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:06:02.486 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.23.128.65"
port: 40287
}
bound_http_address {
host: "127.23.128.65"
port: 40873
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.23.128.66"
port: 33049
}
bound_http_address {
host: "127.23.128.66"
port: 34241
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.23.128.67"
port: 33303
}
bound_http_address {
host: "127.23.128.67"
port: 44163
}
}
}
14:06:02.487 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
14:06:02.647 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52) [main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41) [main/:?]
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43) [main/:?]
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155) [main/:?]
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299) [main/:?]
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109) [main/:?]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) [junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
14:06:02.674 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
14:06:02.798 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskqLh0aI/test-tmp
14:06:02.799 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:06:02.799 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskqLh0aI/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:06:02.801 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
14:06:03.121 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:06:03.121 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:06:03.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:03.124502 24788 mini_oidc.cc:150] Starting JWKS server
14:06:03.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:03.128057 24788 webserver.cc:480] Webserver started at http://127.0.0.1:35989/ using document root <none> and password file <none>
14:06:03.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:03.128561 24788 mini_oidc.cc:186] Starting OIDC Discovery server
14:06:03.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:03.130436 24788 webserver.cc:480] Webserver started at http://127.0.0.1:46877/ using document root <none> and password file <none>
14:06:03.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
14:06:03.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/principal' for realm 'KRBTEST.COM',
14:06:03.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
14:06:03.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:03 dist-test-slave-sg9n krb5kdc[24800](info): setting up network...
14:06:03.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
14:06:03.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:03 dist-test-slave-sg9n krb5kdc[24800](info): set up 2 sockets
14:06:03.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:03 dist-test-slave-sg9n krb5kdc[24800](info): commencing operation
14:06:03.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
14:06:03.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:03.449802 24777 heartbeater.cc:499] Master 127.23.128.124:45273 was elected leader, sending a full tablet report...
14:06:05.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:05.252352 24788 mini_kdc.cc:121] Time spent starting KDC: real 2.086s user 0.007s sys 0.012s
14:06:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
14:06:05.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:06:05.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
14:06:05.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
14:06:05.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:06:05.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
14:06:05.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
14:06:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:06:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
14:06:05.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:06:05.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/test-user.keytab.
14:06:05.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/test-user.keytab.
14:06:05.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:05 dist-test-slave-sg9n krb5kdc[24800](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649165, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:05.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:06:05.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-08-31T14:06:05Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:06:05.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-08-31T14:06:05Z Disabled control of system clock
14:06:05.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.24.53.62@KRBTEST.COM; defaulting to no policy
14:06:05.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:05.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.24.53.62@KRBTEST.COM" created.
14:06:05.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:05.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.53.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:05.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.53.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:05.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.24.53.62@KRBTEST.COM; defaulting to no policy
14:06:05.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:05.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.24.53.62@KRBTEST.COM" created.
14:06:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.24.53.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.24.53.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:05.477629 24788 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:06:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:06:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/wal
14:06:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/data
14:06:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:06:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:06:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:06:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/logs
14:06:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/data/info.pb
14:06:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:06:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:06:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:06:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.53.62:39447
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.53.62
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.53.62
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.53.20:36139
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.53.62:39447,127.24.53.61:38795,127.24.53.60:38575
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35989/jwks.json
14:06:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:06:05.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:06:05.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:05.801537 24817 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:06:05.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:05.802313 24817 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:06:05.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:05.803205 24817 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:06:05.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:05.835825 24817 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:06:05.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:05.836051 24817 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:06:05.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:05.836320 24817 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:06:05.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:05.836519 24817 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:06:05.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:05.836709 24817 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:06:05.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:05.836891 24817 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:06:05.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:05.837031 24817 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35989/jwks.json
14:06:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:05.872490 24817 master_runner.cc:386] Master server non-default flags:
14:06:05.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:05.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.53.20:36139
14:06:05.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:05.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:05.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/data
14:06:05.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/wal
14:06:05.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:06:05.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:05.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.53.62:39447,127.24.53.61:38795,127.24.53.60:38575
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.53.62
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.53.62:39447
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35989/jwks.json
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/data/info.pb
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.53.62
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/logs
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 13:43:23 UTC on 5fd53c4cbb9d
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7833
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:05.873728 24817 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:06:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:05.874758 24817 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:06:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:05.885041 24823 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:07.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:05.885679 24824 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:07.229250 24826 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:07.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:07.232563 24817 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.347s user 0.414s sys 0.889s
14:06:07.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:07.232717 24825 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1341 milliseconds
14:06:07.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:07.232954 24817 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.348s user 0.415s sys 0.889s
14:06:07.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.233253 24817 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:06:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.234357 24817 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:06:07.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.236925 24817 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:06:07.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.238523 24817 hybrid_clock.cc:648] HybridClock initialized: now 1756649167238497 us; error 63 us; skew 500 ppm
14:06:07.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.239352 24817 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:06:07.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:07 dist-test-slave-sg9n krb5kdc[24800](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649167, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:07.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.247007 24817 init.cc:377] Logged in from keytab as kudu/127.24.53.62@KRBTEST.COM (short username kudu)
14:06:07.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.253466 24817 webserver.cc:480] Webserver started at http://127.24.53.62:40963/ using document root <none> and password file <none>
14:06:07.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.254767 24817 fs_manager.cc:362] Metadata directory not provided
14:06:07.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.255069 24817 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:06:07.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.255652 24817 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:06:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.263110 24817 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/data/instance:
14:06:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "99305c0f0bbf4ff9934fffd110334b9d"
14:06:07.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:07 on dist-test-slave-sg9n"
14:06:07.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.264755 24817 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/wal/instance:
14:06:07.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "99305c0f0bbf4ff9934fffd110334b9d"
14:06:07.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:07 on dist-test-slave-sg9n"
14:06:07.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.273293 24817 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.003s
14:06:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.279305 24834 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:06:07.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.280635 24817 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
14:06:07.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.280995 24817 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/data,/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/wal
14:06:07.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "99305c0f0bbf4ff9934fffd110334b9d"
14:06:07.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:07 on dist-test-slave-sg9n"
14:06:07.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.281327 24817 fs_report.cc:389] FS layout report
14:06:07.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:06:07.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/wal
14:06:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/wal
14:06:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/data/data
14:06:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:06:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:06:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:06:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:06:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:06:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:06:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:06:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:06:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:06:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:06:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:06:07.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.349529 24817 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:06:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.355581 24817 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:06:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.356081 24817 kserver.cc:163] Server-wide thread pool size limit: 3276
14:06:07.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:07.368007 24633 debug-util.cc:398] Leaking SignalData structure 0x7b08000acb60 after lost signal to thread 24507
14:06:07.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:07.368866 24633 debug-util.cc:398] Leaking SignalData structure 0x7b08000acc40 after lost signal to thread 24636
14:06:07.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.446099 24817 rpc_server.cc:307] RPC server started. Bound to: 127.24.53.62:39447
14:06:07.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.446235 24885 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.53.62:39447 every 8 connection(s)
14:06:07.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.448989 24817 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/data/info.pb
14:06:07.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.452195 24788 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu as pid 24817
14:06:07.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.452814 24788 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/wal/instance
14:06:07.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.456697 24886 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:06:07.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.24.53.61@KRBTEST.COM; defaulting to no policy
14:06:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.24.53.61@KRBTEST.COM" created.
14:06:07.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.505098 24886 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } has no permanent_uuid. Determining permanent_uuid...
14:06:07.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:07.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.53.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:07.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.53.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:07.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:07.560672 24788 mini_kdc.cc:279] Time spent creating service keytab for kudu/127.24.53.61: real 0.101s user 0.003s sys 0.010s
14:06:07.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:07 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649167, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.62@KRBTEST.COM for kudu/127.24.53.62@KRBTEST.COM
14:06:07.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.24.53.61@KRBTEST.COM; defaulting to no policy
14:06:07.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:07.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.24.53.61@KRBTEST.COM" created.
14:06:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.586326 24886 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } has no permanent_uuid. Determining permanent_uuid...
14:06:07.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:07.589550 24837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.53.61:38795: connect: Connection refused (error 111)
14:06:07.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:07.594550 24886 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.53.61:38795: Network error: Client connection negotiation failed: client connection to 127.24.53.61:38795: connect: Connection refused (error 111)
14:06:07.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:07.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.24.53.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:07.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.24.53.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:07.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.604714 24788 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:06:07.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:06:07.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/wal
14:06:07.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/data
14:06:07.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:06:07.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:06:07.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:06:07.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:07.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:07.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:07.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/logs
14:06:07.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/data/info.pb
14:06:07.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:07.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:06:07.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:06:07.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:06:07.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:06:07.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:07.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:07.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:07.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:06:07.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:06:07.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:07.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:07.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.53.61:38795
14:06:07.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.53.61
14:06:07.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:07.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab
14:06:07.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.53.61
14:06:07.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:07.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:06:07.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:06:07.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:06:07.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.53.20:36139
14:06:07.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:07.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:07.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:07.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:07.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.53.62:39447,127.24.53.61:38795,127.24.53.60:38575
14:06:07.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:07.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35989/jwks.json
14:06:07.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:06:07.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:06:07.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.644165 24886 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } attempt: 1
14:06:07.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:07.648303 24886 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.53.61:38795: Network error: Client connection negotiation failed: client connection to 127.24.53.61:38795: connect: Connection refused (error 111)
14:06:07.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.716715 24886 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } attempt: 2
14:06:07.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:07.722986 24886 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.53.61:38795: Network error: Client connection negotiation failed: client connection to 127.24.53.61:38795: connect: Connection refused (error 111)
14:06:07.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.814436 24886 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } attempt: 3
14:06:07.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:07.819314 24886 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.53.61:38795: Network error: Client connection negotiation failed: client connection to 127.24.53.61:38795: connect: Connection refused (error 111)
14:06:07.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:07.960841 24895 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:06:07.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:07.961525 24895 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:06:07.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:07.962275 24895 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:06:07.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:07.962756 24886 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } attempt: 4
14:06:07.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:07.968307 24886 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.53.61:38795: Network error: Client connection negotiation failed: client connection to 127.24.53.61:38795: connect: Connection refused (error 111)
14:06:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:08.001736 24895 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:06:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:08.001996 24895 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:06:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:08.002271 24895 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:06:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:08.002513 24895 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:06:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:08.002825 24895 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:06:08.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:08.003042 24895 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:06:08.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:08.003216 24895 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35989/jwks.json
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:08.040588 24895 master_runner.cc:386] Master server non-default flags:
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.53.20:36139
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/data
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/wal
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.53.62:39447,127.24.53.61:38795,127.24.53.60:38575
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.53.61
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.53.61:38795
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35989/jwks.json
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/data/info.pb
14:06:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.53.61
14:06:08.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:08.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:06:08.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:08.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:06:08.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:06:08.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:08.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/logs
14:06:08.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:08.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:06:08.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:06:08.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:06:08.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:06:08.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:06:08.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:06:08.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 13:43:23 UTC on 5fd53c4cbb9d
14:06:08.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7833
14:06:08.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:06:08.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:08.041805 24895 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:06:08.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:08.042936 24895 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:06:08.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:08.054917 24903 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:08.267722 24886 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } attempt: 5
14:06:08.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:08.273602 24886 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.53.61:38795: Network error: Client connection negotiation failed: client connection to 127.24.53.61:38795: connect: Connection refused (error 111)
14:06:08.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:08.554857 24773 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac040 after lost signal to thread 24648
14:06:08.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:08.555949 24773 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7f40 after lost signal to thread 24776
14:06:08.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:08.821033 24886 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } attempt: 6
14:06:08.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:08.859647 24886 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.53.61:38795: Network error: Client connection negotiation failed: client connection to 127.24.53.61:38795: connect: Connection refused (error 111)
14:06:09.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:08.055899 24904 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:09.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:09.456236 24902 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 24895
14:06:09.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:09.923914 24886 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } attempt: 7
14:06:09.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:09.919085 24895 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.866s user 0.667s sys 1.109s
14:06:09.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:09.931944 24895 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.879s user 0.667s sys 1.109s
14:06:09.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:09.969902 24895 server_base.cc:1047] running on GCE node
14:06:09.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:09.971313 24886 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.53.61:38795: Network error: Client connection negotiation failed: client connection to 127.24.53.61:38795: connect: Connection refused (error 111)
14:06:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:09.972759 24912 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:09.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:09.974040 24895 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:06:09.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:09.977007 24895 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:06:09.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:09.978454 24895 hybrid_clock.cc:648] HybridClock initialized: now 1756649169978403 us; error 60 us; skew 500 ppm
14:06:09.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:09.979509 24895 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:06:09.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:09 dist-test-slave-sg9n krb5kdc[24800](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649169, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:09.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:09.986464 24895 init.cc:377] Logged in from keytab as kudu/127.24.53.61@KRBTEST.COM (short username kudu)
14:06:09.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:09.992564 24895 webserver.cc:480] Webserver started at http://127.24.53.61:36939/ using document root <none> and password file <none>
14:06:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:09.993965 24895 fs_manager.cc:362] Metadata directory not provided
14:06:09.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:09.994266 24895 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:06:09.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:09.994875 24895 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:06:10.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.002081 24895 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/data/instance:
14:06:10.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae15ba7f491b4bd19bcb57b0ed510483"
14:06:10.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:09 on dist-test-slave-sg9n"
14:06:10.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.003717 24895 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/wal/instance:
14:06:10.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae15ba7f491b4bd19bcb57b0ed510483"
14:06:10.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:09 on dist-test-slave-sg9n"
14:06:10.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.013579 24895 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.009s sys 0.001s
14:06:10.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.021428 24919 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:06:10.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.022827 24895 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.002s
14:06:10.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.023252 24895 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/data,/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/wal
14:06:10.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae15ba7f491b4bd19bcb57b0ed510483"
14:06:10.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:09 on dist-test-slave-sg9n"
14:06:10.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.023726 24895 fs_report.cc:389] FS layout report
14:06:10.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:06:10.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/wal
14:06:10.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/wal
14:06:10.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/data/data
14:06:10.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:06:10.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:06:10.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:06:10.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:06:10.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:06:10.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:06:10.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:06:10.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:06:10.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:06:10.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:06:10.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:06:10.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.096760 24895 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:06:10.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.109229 24895 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:06:10.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.109850 24895 kserver.cc:163] Server-wide thread pool size limit: 3276
14:06:10.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.206902 24895 rpc_server.cc:307] RPC server started. Bound to: 127.24.53.61:38795
14:06:10.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.206969 24971 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.53.61:38795 every 8 connection(s)
14:06:10.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.210055 24895 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/data/info.pb
14:06:10.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.216465 24972 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:06:10.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.216964 24788 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu as pid 24895
14:06:10.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.217454 24788 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/wal/instance
14:06:10.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.228247 24972 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } has no permanent_uuid. Determining permanent_uuid...
14:06:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.24.53.60@KRBTEST.COM; defaulting to no policy
14:06:10.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:10.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.24.53.60@KRBTEST.COM" created.
14:06:10.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:10 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649169, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.61@KRBTEST.COM for kudu/127.24.53.62@KRBTEST.COM
14:06:10.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:10.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.53.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:10.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.53.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:10.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.276707 24972 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } has no permanent_uuid. Determining permanent_uuid...
14:06:10.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.24.53.60@KRBTEST.COM; defaulting to no policy
14:06:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.24.53.60@KRBTEST.COM" created.
14:06:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:10 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649169, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.61@KRBTEST.COM for kudu/127.24.53.61@KRBTEST.COM
14:06:10.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.308259 24972 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } has no permanent_uuid. Determining permanent_uuid...
14:06:10.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:10.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.24.53.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:10.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.24.53.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:10.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.311193 24788 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:06:10.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:06:10.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/wal
14:06:10.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/data
14:06:10.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:06:10.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:06:10.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/logs
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/data/info.pb
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.53.60:38575
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.53.60
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.53.60
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.53.20:36139
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.53.62:39447,127.24.53.61:38795,127.24.53.60:38575
14:06:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35989/jwks.json
14:06:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:06:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:06:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:10.311594 24924 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.53.60:38575: connect: Connection refused (error 111)
14:06:10.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:10.317977 24972 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.53.60:38575: Network error: Client connection negotiation failed: client connection to 127.24.53.60:38575: connect: Connection refused (error 111)
14:06:10.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.367456 24972 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } attempt: 1
14:06:10.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:10.372174 24972 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.53.60:38575: Network error: Client connection negotiation failed: client connection to 127.24.53.60:38575: connect: Connection refused (error 111)
14:06:10.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.440752 24972 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } attempt: 2
14:06:10.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:10.446895 24972 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.53.60:38575: Network error: Client connection negotiation failed: client connection to 127.24.53.60:38575: connect: Connection refused (error 111)
14:06:10.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.538329 24972 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } attempt: 3
14:06:10.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:10.546234 24972 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.53.60:38575: Network error: Client connection negotiation failed: client connection to 127.24.53.60:38575: connect: Connection refused (error 111)
14:06:10.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:10.653822 24982 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:06:10.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:10.654474 24982 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:06:10.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:10.655238 24982 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:06:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.689702 24972 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } attempt: 4
14:06:10.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:10.695497 24972 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.53.60:38575: Network error: Client connection negotiation failed: client connection to 127.24.53.60:38575: connect: Connection refused (error 111)
14:06:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:10.708552 24982 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:06:10.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:10.709177 24982 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:06:10.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:10.709795 24982 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:06:10.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:10.710273 24982 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:06:10.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:10.710718 24982 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:06:10.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:10.711159 24982 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:06:10.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:10.711524 24982 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35989/jwks.json
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.764684 24982 master_runner.cc:386] Master server non-default flags:
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.53.20:36139
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/data
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/wal
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.53.62:39447,127.24.53.61:38795,127.24.53.60:38575
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.53.60
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.53.60:38575
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35989/jwks.json
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/data/info.pb
14:06:10.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.53.60
14:06:10.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:10.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:06:10.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:10.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:06:10.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:06:10.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:10.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/logs
14:06:10.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:10.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:06:10.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:06:10.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:06:10.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:06:10.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:06:10.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:06:10.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 13:43:23 UTC on 5fd53c4cbb9d
14:06:10.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7833
14:06:10.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:06:10.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.766331 24982 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:06:10.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.767771 24982 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:06:10.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:10.791100 24990 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:10.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:10.968904 24633 debug-util.cc:398] Leaking SignalData structure 0x7b08000b3ce0 after lost signal to thread 24507
14:06:10.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:10.970449 24633 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6ba0 after lost signal to thread 24636
14:06:10.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:10.994928 24972 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } attempt: 5
14:06:11.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:11.001157 24972 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.53.60:38575: Network error: Client connection negotiation failed: client connection to 127.24.53.60:38575: connect: Connection refused (error 111)
14:06:11.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:11.548635 24972 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } attempt: 6
14:06:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:11.585104 24972 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.53.60:38575: Network error: Client connection negotiation failed: client connection to 127.24.53.60:38575: connect: Connection refused (error 111)
14:06:12.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.061769 24886 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } attempt: 8
14:06:12.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:12 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649167, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.62@KRBTEST.COM for kudu/127.24.53.61@KRBTEST.COM
14:06:12.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.124123 24886 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } has no permanent_uuid. Determining permanent_uuid...
14:06:12.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:12.148824 24886 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.53.60:38575: Network error: Client connection negotiation failed: client connection to 127.24.53.60:38575: connect: Connection refused (error 111)
14:06:12.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.214347 24886 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } attempt: 1
14:06:12.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:12.222343 24886 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.53.60:38575: Network error: Client connection negotiation failed: client connection to 127.24.53.60:38575: connect: Connection refused (error 111)
14:06:12.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.275882 24886 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } attempt: 2
14:06:12.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:12.283337 24886 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.53.60:38575: Network error: Client connection negotiation failed: client connection to 127.24.53.60:38575: connect: Connection refused (error 111)
14:06:12.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:12.191507 24988 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 24982
14:06:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:12.329598 24988 kernel_stack_watchdog.cc:198] Thread 24982 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 396ms:
14:06:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:06:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:06:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:06:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:06:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:10.793141 24989 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:12.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:12.330693 24982 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.538s user 0.465s sys 1.021s
14:06:12.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:12.331120 24982 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.539s user 0.465s sys 1.021s
14:06:12.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:12.331617 24991 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1538 milliseconds
14:06:12.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:12.332962 24993 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:12.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.332949 24982 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:06:12.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.336206 24982 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:06:12.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.338809 24982 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:06:12.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.340200 24982 hybrid_clock.cc:648] HybridClock initialized: now 1756649172340167 us; error 36 us; skew 500 ppm
14:06:12.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.341022 24982 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:06:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:12 dist-test-slave-sg9n krb5kdc[24800](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649172, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:12.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.347069 24982 init.cc:377] Logged in from keytab as kudu/127.24.53.60@KRBTEST.COM (short username kudu)
14:06:12.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.351914 24982 webserver.cc:480] Webserver started at http://127.24.53.60:32857/ using document root <none> and password file <none>
14:06:12.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.352871 24982 fs_manager.cc:362] Metadata directory not provided
14:06:12.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.353087 24982 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:06:12.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.353689 24982 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:06:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.358119 24982 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/data/instance:
14:06:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9b08d0ced4ac40308f7432e555465203"
14:06:12.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:12 on dist-test-slave-sg9n"
14:06:12.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.359395 24982 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/wal/instance:
14:06:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9b08d0ced4ac40308f7432e555465203"
14:06:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:12 on dist-test-slave-sg9n"
14:06:12.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.359805 24886 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } attempt: 3
14:06:12.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:12.363360 24886 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.24.53.60:38575: Network error: Client connection negotiation failed: client connection to 127.24.53.60:38575: connect: Connection refused (error 111)
14:06:12.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.367108 24982 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
14:06:12.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.373080 25006 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:06:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.374105 24982 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
14:06:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.374444 24982 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/data,/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/wal
14:06:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9b08d0ced4ac40308f7432e555465203"
14:06:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:12 on dist-test-slave-sg9n"
14:06:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.374933 24982 fs_report.cc:389] FS layout report
14:06:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:06:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/wal
14:06:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/wal
14:06:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/data/data
14:06:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:06:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:06:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:06:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:06:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:06:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:06:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:06:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:06:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:06:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:06:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:06:12.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.435642 24982 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:06:12.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.440856 24982 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:06:12.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.441282 24982 kserver.cc:163] Server-wide thread pool size limit: 3276
14:06:12.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.513290 24982 rpc_server.cc:307] RPC server started. Bound to: 127.24.53.60:38575
14:06:12.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.513394 25057 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.53.60:38575 every 8 connection(s)
14:06:12.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.516426 24982 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/data/info.pb
14:06:12.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.518811 24886 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } attempt: 4
14:06:12.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.522044 25058 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:06:12.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.526628 24788 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu as pid 24982
14:06:12.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.527146 24788 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/wal/instance
14:06:12.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.547420 25058 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } has no permanent_uuid. Determining permanent_uuid...
14:06:12.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:12 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649167, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.62@KRBTEST.COM for kudu/127.24.53.60@KRBTEST.COM
14:06:12.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:12 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649172, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.60@KRBTEST.COM for kudu/127.24.53.62@KRBTEST.COM
14:06:12.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:12 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649165, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.53.62@KRBTEST.COM
14:06:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.583568 25058 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } has no permanent_uuid. Determining permanent_uuid...
14:06:12.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.583902 24886 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d: Bootstrap starting.
14:06:12.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.593935 24886 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d: Neither blocks nor log segments found. Creating new log.
14:06:12.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.596562 24886 log.cc:826] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d: Log is configured to *not* fsync() on all Append() calls
14:06:12.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:12 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649172, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.60@KRBTEST.COM for kudu/127.24.53.61@KRBTEST.COM
14:06:12.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:12 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649165, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.53.61@KRBTEST.COM
14:06:12.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.604321 24886 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d: No bootstrap required, opened a new log
14:06:12.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.613441 25058 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } has no permanent_uuid. Determining permanent_uuid...
14:06:12.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:12 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649165, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.24.53.60@KRBTEST.COM
14:06:12.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:12 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649172, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.60@KRBTEST.COM for kudu/127.24.53.60@KRBTEST.COM
14:06:12.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.630419 24886 raft_consensus.cc:357] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } }
14:06:12.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.631292 24886 raft_consensus.cc:383] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:06:12.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.631587 24886 raft_consensus.cc:738] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 99305c0f0bbf4ff9934fffd110334b9d, State: Initialized, Role: FOLLOWER
14:06:12.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.632527 24886 consensus_queue.cc:260] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } }
14:06:12.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.636360 25068 sys_catalog.cc:455] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } } }
14:06:12.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.637388 25068 sys_catalog.cc:458] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [sys.catalog]: This master's current role is: FOLLOWER
14:06:12.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.637655 24886 sys_catalog.cc:564] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [sys.catalog]: configured and running, proceeding with master startup.
14:06:12.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.645627 24972 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } attempt: 7
14:06:12.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.649945 25058 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203: Bootstrap starting.
14:06:12.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.660768 25058 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203: Neither blocks nor log segments found. Creating new log.
14:06:12.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.663146 25058 log.cc:826] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203: Log is configured to *not* fsync() on all Append() calls
14:06:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:12 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649169, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.61@KRBTEST.COM for kudu/127.24.53.60@KRBTEST.COM
14:06:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.689369 25058 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203: No bootstrap required, opened a new log
14:06:12.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.715127 24972 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483: Bootstrap starting.
14:06:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.722651 24972 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483: Neither blocks nor log segments found. Creating new log.
14:06:12.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.725203 24972 log.cc:826] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483: Log is configured to *not* fsync() on all Append() calls
14:06:12.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:12.735394 25081 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:06:12.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:12.735749 25081 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:06:12.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.743762 25058 raft_consensus.cc:357] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } }
14:06:12.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.744664 25058 raft_consensus.cc:383] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:06:12.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.744946 25058 raft_consensus.cc:738] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9b08d0ced4ac40308f7432e555465203, State: Initialized, Role: FOLLOWER
14:06:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.746284 25058 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } }
14:06:12.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.751662 24972 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483: No bootstrap required, opened a new log
14:06:12.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.694398 24817 master_runner.cc:186] Error getting master registration for 127.24.53.61:38795: OK, instance_id { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" instance_seqno: 1756649170179659 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:06:12.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.757890 24817 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:06:12.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.768349 25084 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } } }
14:06:12.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.768934 25084 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [sys.catalog]: This master's current role is: FOLLOWER
14:06:12.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.771250 25058 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [sys.catalog]: configured and running, proceeding with master startup.
14:06:12.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.789958 25068 raft_consensus.cc:491] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:06:12.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.790495 25068 raft_consensus.cc:513] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } }
14:06:12.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.805424 25068 leader_election.cc:290] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ae15ba7f491b4bd19bcb57b0ed510483 (127.24.53.61:38795), 9b08d0ced4ac40308f7432e555465203 (127.24.53.60:38575)
14:06:12.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.806185 25033 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "99305c0f0bbf4ff9934fffd110334b9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9b08d0ced4ac40308f7432e555465203" is_pre_election: true
14:06:12.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.807070 25033 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 99305c0f0bbf4ff9934fffd110334b9d in term 0.
14:06:12.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.808503 24838 leader_election.cc:304] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 99305c0f0bbf4ff9934fffd110334b9d, 9b08d0ced4ac40308f7432e555465203; no voters:
14:06:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.810170 25068 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 0 FOLLOWER]: Leader pre-election won for term 1
14:06:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.811177 25068 raft_consensus.cc:491] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:06:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.811230 24972 raft_consensus.cc:357] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } }
14:06:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.811983 24972 raft_consensus.cc:383] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:06:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.812208 25068 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 0 FOLLOWER]: Advancing to term 1
14:06:12.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.814234 24972 raft_consensus.cc:738] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ae15ba7f491b4bd19bcb57b0ed510483, State: Initialized, Role: FOLLOWER
14:06:12.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.815402 24972 consensus_queue.cc:260] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } }
14:06:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.825819 24947 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "99305c0f0bbf4ff9934fffd110334b9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" is_pre_election: true
14:06:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.826574 24947 raft_consensus.cc:2466] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 99305c0f0bbf4ff9934fffd110334b9d in term 0.
14:06:12.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.828840 25094 sys_catalog.cc:455] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } } }
14:06:12.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.829648 24972 sys_catalog.cc:564] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [sys.catalog]: configured and running, proceeding with master startup.
14:06:12.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.830648 25094 sys_catalog.cc:458] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [sys.catalog]: This master's current role is: FOLLOWER
14:06:12.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.835633 25068 raft_consensus.cc:513] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } }
14:06:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.839650 25096 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:06:12.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.845335 24947 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "99305c0f0bbf4ff9934fffd110334b9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ae15ba7f491b4bd19bcb57b0ed510483"
14:06:12.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.845873 24947 raft_consensus.cc:3058] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [term 0 FOLLOWER]: Advancing to term 1
14:06:12.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.851692 24947 raft_consensus.cc:2466] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 99305c0f0bbf4ff9934fffd110334b9d in term 1.
14:06:12.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.852674 24837 leader_election.cc:304] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 99305c0f0bbf4ff9934fffd110334b9d, ae15ba7f491b4bd19bcb57b0ed510483; no voters:
14:06:12.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.867419 25068 leader_election.cc:290] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [CANDIDATE]: Term 1 election: Requested vote from peers ae15ba7f491b4bd19bcb57b0ed510483 (127.24.53.61:38795), 9b08d0ced4ac40308f7432e555465203 (127.24.53.60:38575)
14:06:12.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.867813 25033 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "99305c0f0bbf4ff9934fffd110334b9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9b08d0ced4ac40308f7432e555465203"
14:06:12.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:12.868052 24840 tablet.cc:2378] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:06:12.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.868562 25033 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [term 0 FOLLOWER]: Advancing to term 1
14:06:12.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:12.868989 24926 tablet.cc:2378] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:06:12.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.868788 25068 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 1 FOLLOWER]: Leader election won for term 1
14:06:12.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.870049 25068 raft_consensus.cc:695] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 1 LEADER]: Becoming Leader. State: Replica: 99305c0f0bbf4ff9934fffd110334b9d, State: Running, Role: LEADER
14:06:12.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.871577 25068 consensus_queue.cc:237] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } }
14:06:12.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.878562 25033 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 99305c0f0bbf4ff9934fffd110334b9d in term 1.
14:06:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.892107 25099 sys_catalog.cc:455] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 99305c0f0bbf4ff9934fffd110334b9d. Latest consensus state: current_term: 1 leader_uuid: "99305c0f0bbf4ff9934fffd110334b9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } } }
14:06:12.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.895071 25099 sys_catalog.cc:458] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [sys.catalog]: This master's current role is: LEADER
14:06:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.852715 24982 master_runner.cc:186] Error getting master registration for 127.24.53.61:38795: OK, instance_id { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" instance_seqno: 1756649170179659 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:06:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.896348 24982 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:06:12.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.904178 25104 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:06:12.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.912195 25104 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:06:12.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.944130 24947 raft_consensus.cc:1273] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [term 1 FOLLOWER]: Refusing update from remote peer 99305c0f0bbf4ff9934fffd110334b9d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:06:12.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.945744 25099 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:06:12.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.947614 25033 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [term 1 FOLLOWER]: Refusing update from remote peer 99305c0f0bbf4ff9934fffd110334b9d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:06:12.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:12.951005 25012 tablet.cc:2378] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:06:12.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:12.951869 25110 catalog_manager.cc:1560] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:06:12.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:12.952167 25110 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:06:12.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.952668 25099 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:06:12.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.991886 25094 sys_catalog.cc:455] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 99305c0f0bbf4ff9934fffd110334b9d. Latest consensus state: current_term: 1 leader_uuid: "99305c0f0bbf4ff9934fffd110334b9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } } }
14:06:12.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:12.994832 25094 sys_catalog.cc:458] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [sys.catalog]: This master's current role is: FOLLOWER
14:06:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.005182 25112 mvcc.cc:204] Tried to move back new op lower bound from 7195235012350410752 to 7195235012097138688. Current Snapshot: MvccSnapshot[applied={T|T < 7195235012350410752}]
14:06:13.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.006493 25084 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 99305c0f0bbf4ff9934fffd110334b9d. Latest consensus state: current_term: 1 leader_uuid: "99305c0f0bbf4ff9934fffd110334b9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } } }
14:06:13.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.007347 25084 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [sys.catalog]: This master's current role is: FOLLOWER
14:06:13.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.010473 25068 sys_catalog.cc:455] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "99305c0f0bbf4ff9934fffd110334b9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } } }
14:06:13.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.011180 25068 sys_catalog.cc:458] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [sys.catalog]: This master's current role is: LEADER
14:06:13.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.016973 25094 sys_catalog.cc:455] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "99305c0f0bbf4ff9934fffd110334b9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } } }
14:06:13.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.017565 25094 sys_catalog.cc:458] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [sys.catalog]: This master's current role is: FOLLOWER
14:06:13.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.018497 25104 catalog_manager.cc:1349] Generated new cluster ID: b14b064117c2418a8ecf7e52655578ff
14:06:13.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.018795 25104 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:06:13.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.020629 25099 sys_catalog.cc:455] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "99305c0f0bbf4ff9934fffd110334b9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } } }
14:06:13.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.021260 25099 sys_catalog.cc:458] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [sys.catalog]: This master's current role is: LEADER
14:06:13.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.039275 25084 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "99305c0f0bbf4ff9934fffd110334b9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } } }
14:06:13.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.039958 25084 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [sys.catalog]: This master's current role is: FOLLOWER
14:06:13.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.068892 25104 catalog_manager.cc:1372] Generated new certificate authority record
14:06:13.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.074169 25104 catalog_manager.cc:1506] Loading token signing keys...
14:06:13.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.104029 25104 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d: Generated new TSK 0
14:06:13.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.105422 25104 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:06:13.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.24.53.1@KRBTEST.COM; defaulting to no policy
14:06:13.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:13.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.24.53.1@KRBTEST.COM" created.
14:06:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.53.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.53.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:13.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.24.53.1@KRBTEST.COM; defaulting to no policy
14:06:13.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:13.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.24.53.1@KRBTEST.COM" created.
14:06:13.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:13.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.24.53.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:13.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.24.53.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:13.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.240109 24788 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:06:13.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:06:13.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-0/wal
14:06:13.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-0/data
14:06:13.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:06:13.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:06:13.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:06:13.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:13.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:13.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:13.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-0/logs
14:06:13.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-0/data/info.pb
14:06:13.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:13.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:06:13.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:06:13.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:06:13.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:06:13.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:13.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:13.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:13.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:06:13.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:06:13.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.53.1:0
14:06:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.53.1
14:06:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.53.1
14:06:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.53.62:39447,127.24.53.61:38795,127.24.53.60:38575
14:06:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab
14:06:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.53.1
14:06:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:06:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:06:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:06:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.53.20:36139
14:06:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:06:13.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:13.774289 25125 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:06:13.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:13.775331 25125 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:06:13.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:13.776549 25125 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:06:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:13.835700 25125 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:06:13.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:13.837368 25125 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.53.1
14:06:13.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.864046 25096 catalog_manager.cc:1261] Loaded cluster ID: b14b064117c2418a8ecf7e52655578ff
14:06:13.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.864593 25096 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203: loading cluster ID for follower catalog manager: success
14:06:13.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.882361 25096 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203: acquiring CA information for follower catalog manager: success
14:06:13.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.891974 25096 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:06:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.902966 25125 tablet_server_runner.cc:78] Tablet server non-default flags:
14:06:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.53.20:36139
14:06:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-0/data
14:06:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-0/wal
14:06:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab
14:06:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.53.1
14:06:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.53.1:0
14:06:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:06:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:06:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:06:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-0/data/info.pb
14:06:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.53.1
14:06:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.53.62:39447,127.24.53.61:38795,127.24.53.60:38575
14:06:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:06:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:06:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:06:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.53.1
14:06:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-0/logs
14:06:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:06:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:06:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:06:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:06:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:06:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:06:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 13:43:23 UTC on 5fd53c4cbb9d
14:06:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7833
14:06:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:06:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.904667 25125 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:06:13.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.906111 25125 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:06:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:13.941211 25132 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.959375 25110 catalog_manager.cc:1261] Loaded cluster ID: b14b064117c2418a8ecf7e52655578ff
14:06:13.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.959728 25110 catalog_manager.cc:1554] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483: loading cluster ID for follower catalog manager: success
14:06:13.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:13.979781 25110 catalog_manager.cc:1576] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483: acquiring CA information for follower catalog manager: success
14:06:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:14.007958 25110 catalog_manager.cc:1604] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:06:15.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:15.484274 24141 debug-util.cc:398] Leaking SignalData structure 0x7b080009d340 after lost signal to thread 24078
14:06:15.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:15.485066 24141 debug-util.cc:398] Leaking SignalData structure 0x7b08000875e0 after lost signal to thread 24144
14:06:15.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:15.329187 25131 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 25125
14:06:15.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:15.744032 25136 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:15.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:13.946002 25133 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:15.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:15.743392 25125 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.800s user 0.225s sys 0.383s
14:06:15.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:15.750293 25125 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.807s user 0.225s sys 0.383s
14:06:15.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:15.755198 25134 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1802 milliseconds
14:06:15.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:15.755247 25125 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:06:15.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:15.756639 25125 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:06:15.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:15.790666 25125 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:06:15.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:15.796948 25125 hybrid_clock.cc:648] HybridClock initialized: now 1756649175796853 us; error 79 us; skew 500 ppm
14:06:15.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:15.798166 25125 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:06:15.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:15 dist-test-slave-sg9n krb5kdc[24800](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649175, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:15.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:15.811002 25125 init.cc:377] Logged in from keytab as kudu/127.24.53.1@KRBTEST.COM (short username kudu)
14:06:15.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:15.847373 25125 webserver.cc:480] Webserver started at http://127.24.53.1:41369/ using document root <none> and password file <none>
14:06:15.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:15.848696 25125 fs_manager.cc:362] Metadata directory not provided
14:06:15.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:15.849000 25125 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:06:15.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:15.849574 25125 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:06:15.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:15.862082 25125 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-0/data/instance:
14:06:15.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a373d4f104c846e99bd61b11d7185280"
14:06:15.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:15 on dist-test-slave-sg9n"
14:06:15.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:15.867625 25125 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-0/wal/instance:
14:06:15.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a373d4f104c846e99bd61b11d7185280"
14:06:15.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:15 on dist-test-slave-sg9n"
14:06:15.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:15.899709 25125 fs_manager.cc:696] Time spent creating directory manager: real 0.031s user 0.011s sys 0.001s
14:06:15.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:15.920037 25148 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:06:15.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:15.921494 25125 fs_manager.cc:730] Time spent opening block manager: real 0.017s user 0.006s sys 0.001s
14:06:15.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:15.921938 25125 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-0/data,/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-0/wal
14:06:15.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a373d4f104c846e99bd61b11d7185280"
14:06:15.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:15 on dist-test-slave-sg9n"
14:06:15.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:15.922407 25125 fs_report.cc:389] FS layout report
14:06:15.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:06:15.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-0/wal
14:06:15.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-0/wal
14:06:15.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-0/data/data
14:06:15.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:06:15.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:06:15.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:06:15.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:06:15.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:06:15.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:06:15.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:06:15.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:06:15.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:06:15.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:06:15.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:06:16.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:16.087348 25125 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:06:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:16.111932 25125 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:06:16.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:16.112526 25125 kserver.cc:163] Server-wide thread pool size limit: 3276
14:06:16.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:16.128463 25125 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:06:16.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:16.145249 25125 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:06:16.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:16.145582 25125 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:06:16.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:16.145982 25125 ts_tablet_manager.cc:610] Registered 0 tablets
14:06:16.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:16.146205 25125 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:06:17.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.108604 25260 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.53.1:34699 every 8 connection(s)
14:06:17.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.109169 25125 rpc_server.cc:307] RPC server started. Bound to: 127.24.53.1:34699
14:06:17.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.112979 25125 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-0/data/info.pb
14:06:17.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.118007 24788 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu as pid 25125
14:06:17.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.118448 24788 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-0/wal/instance
14:06:17.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.24.53.2@KRBTEST.COM; defaulting to no policy
14:06:17.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:17.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.24.53.2@KRBTEST.COM" created.
14:06:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:17 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649175, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.1@KRBTEST.COM for kudu/127.24.53.62@KRBTEST.COM
14:06:17.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:17.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.53.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:17.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.53.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:17.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:17.234117 24788 mini_kdc.cc:279] Time spent creating service keytab for kudu/127.24.53.2: real 0.105s user 0.000s sys 0.015s
14:06:17.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:17 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649175, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.1@KRBTEST.COM for kudu/127.24.53.60@KRBTEST.COM
14:06:17.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:17 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649175, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.1@KRBTEST.COM for kudu/127.24.53.61@KRBTEST.COM
14:06:17.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:17 dist-test-slave-sg9n krb5kdc[24800](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649177, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:17.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.252808 25266 heartbeater.cc:344] Connected to a master server at 127.24.53.62:39447
14:06:17.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.253211 24830 init.cc:295] Successfully reacquired a new kerberos TGT
14:06:17.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.253334 25266 heartbeater.cc:461] Registering TS with master...
14:06:17.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.254848 25266 heartbeater.cc:507] Master 127.24.53.62:39447 requested a full tablet report, sending...
14:06:17.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.258292 24851 ts_manager.cc:194] Registered new tserver with Master: a373d4f104c846e99bd61b11d7185280 (127.24.53.1:34699)
14:06:17.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.261461 24851 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.24.53.1@KRBTEST.COM'} at 127.24.53.1:55985
14:06:17.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.24.53.2@KRBTEST.COM; defaulting to no policy
14:06:17.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:17.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.24.53.2@KRBTEST.COM" created.
14:06:17.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.284935 25261 heartbeater.cc:344] Connected to a master server at 127.24.53.60:38575
14:06:17.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.285316 25261 heartbeater.cc:461] Registering TS with master...
14:06:17.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.285992 25261 heartbeater.cc:507] Master 127.24.53.60:38575 requested a full tablet report, sending...
14:06:17.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.289688 25023 ts_manager.cc:194] Registered new tserver with Master: a373d4f104c846e99bd61b11d7185280 (127.24.53.1:34699)
14:06:17.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.290201 25262 heartbeater.cc:344] Connected to a master server at 127.24.53.61:38795
14:06:17.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.290546 25262 heartbeater.cc:461] Registering TS with master...
14:06:17.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.291318 25262 heartbeater.cc:507] Master 127.24.53.61:38795 requested a full tablet report, sending...
14:06:17.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.294461 24937 ts_manager.cc:194] Registered new tserver with Master: a373d4f104c846e99bd61b11d7185280 (127.24.53.1:34699)
14:06:17.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:17.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.24.53.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:17.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.24.53.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:17.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.310344 24788 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:06:17.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:06:17.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-1/wal
14:06:17.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-1/data
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-1/logs
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-1/data/info.pb
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.53.2:0
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.53.2
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.53.2
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.53.62:39447,127.24.53.61:38795,127.24.53.60:38575
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.53.2
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:06:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:06:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:06:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.53.20:36139
14:06:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:06:17.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:17.782253 25275 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:06:17.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:17.783221 25275 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:06:17.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:17.784631 25275 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:06:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:17.851722 25275 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:06:17.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:17.853569 25275 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.53.2
14:06:17.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.927933 25275 tablet_server_runner.cc:78] Tablet server non-default flags:
14:06:17.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:17.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.53.20:36139
14:06:17.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:17.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:17.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-1/data
14:06:17.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-1/wal
14:06:17.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab
14:06:17.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.53.2
14:06:17.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:17.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:17.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.53.2:0
14:06:17.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:06:17.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:17.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:06:17.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:06:17.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:17.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:17.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-1/data/info.pb
14:06:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.53.2
14:06:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.53.62:39447,127.24.53.61:38795,127.24.53.60:38575
14:06:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:06:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:06:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:06:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.53.2
14:06:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-1/logs
14:06:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:06:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:06:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:06:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:06:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:06:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:06:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 13:43:23 UTC on 5fd53c4cbb9d
14:06:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7833
14:06:17.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:06:17.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.929880 25275 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:06:17.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.931401 25275 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:06:17.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:17.948947 25282 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:18.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:18.266541 25266 heartbeater.cc:499] Master 127.24.53.62:39447 was elected leader, sending a full tablet report...
14:06:19.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:17.953329 25283 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:19.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:17.955672 25285 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:19.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:17.954805 25275 server_base.cc:1047] running on GCE node
14:06:19.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.200223 25275 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:06:19.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.202576 25275 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:06:19.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.204017 25275 hybrid_clock.cc:648] HybridClock initialized: now 1756649179203948 us; error 81 us; skew 500 ppm
14:06:19.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.205080 25275 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:06:19.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:19 dist-test-slave-sg9n krb5kdc[24800](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649179, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.213042 25275 init.cc:377] Logged in from keytab as kudu/127.24.53.2@KRBTEST.COM (short username kudu)
14:06:19.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.218667 25275 webserver.cc:480] Webserver started at http://127.24.53.2:37793/ using document root <none> and password file <none>
14:06:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.219921 25275 fs_manager.cc:362] Metadata directory not provided
14:06:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.220188 25275 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:06:19.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.220716 25275 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:06:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.227375 25275 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-1/data/instance:
14:06:19.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "15cfa4072149430492223ea62613b6fd"
14:06:19.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:19 on dist-test-slave-sg9n"
14:06:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.228863 25275 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-1/wal/instance:
14:06:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "15cfa4072149430492223ea62613b6fd"
14:06:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:19 on dist-test-slave-sg9n"
14:06:19.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.238336 25275 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.006s sys 0.004s
14:06:19.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.245304 25297 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:06:19.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.246421 25275 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
14:06:19.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.246783 25275 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-1/data,/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-1/wal
14:06:19.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "15cfa4072149430492223ea62613b6fd"
14:06:19.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:19 on dist-test-slave-sg9n"
14:06:19.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.247134 25275 fs_report.cc:389] FS layout report
14:06:19.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:06:19.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-1/wal
14:06:19.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-1/wal
14:06:19.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-1/data/data
14:06:19.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:06:19.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:06:19.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:06:19.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:06:19.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:06:19.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:06:19.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:06:19.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:06:19.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:06:19.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:06:19.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:06:19.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.307289 25275 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:06:19.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.313202 25275 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:06:19.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.313740 25275 kserver.cc:163] Server-wide thread pool size limit: 3276
14:06:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.316437 25275 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:06:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.321429 25275 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:06:19.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.321653 25275 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:06:19.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.321895 25275 ts_tablet_manager.cc:610] Registered 0 tablets
14:06:19.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.322078 25275 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:06:19.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.510574 25275 rpc_server.cc:307] RPC server started. Bound to: 127.24.53.2:38155
14:06:19.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.510751 25410 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.53.2:38155 every 8 connection(s)
14:06:19.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.513254 25275 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-1/data/info.pb
14:06:19.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.516237 24788 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu as pid 25275
14:06:19.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.516779 24788 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-1/wal/instance
14:06:19.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:19 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649179, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.2@KRBTEST.COM for kudu/127.24.53.60@KRBTEST.COM
14:06:19.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.578953 25411 heartbeater.cc:344] Connected to a master server at 127.24.53.60:38575
14:06:19.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.579646 25411 heartbeater.cc:461] Registering TS with master...
14:06:19.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.581391 25411 heartbeater.cc:507] Master 127.24.53.60:38575 requested a full tablet report, sending...
14:06:19.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.590598 25023 ts_manager.cc:194] Registered new tserver with Master: 15cfa4072149430492223ea62613b6fd (127.24.53.2:38155)
14:06:19.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.24.53.3@KRBTEST.COM; defaulting to no policy
14:06:19.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:19.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.24.53.3@KRBTEST.COM" created.
14:06:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:19 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649179, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.2@KRBTEST.COM for kudu/127.24.53.61@KRBTEST.COM
14:06:19.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:19 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649179, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.2@KRBTEST.COM for kudu/127.24.53.62@KRBTEST.COM
14:06:19.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.621428 25412 heartbeater.cc:344] Connected to a master server at 127.24.53.61:38795
14:06:19.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.621766 25412 heartbeater.cc:461] Registering TS with master...
14:06:19.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.622792 25412 heartbeater.cc:507] Master 127.24.53.61:38795 requested a full tablet report, sending...
14:06:19.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.625643 24937 ts_manager.cc:194] Registered new tserver with Master: 15cfa4072149430492223ea62613b6fd (127.24.53.2:38155)
14:06:19.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.626178 25415 heartbeater.cc:344] Connected to a master server at 127.24.53.62:39447
14:06:19.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.626463 25415 heartbeater.cc:461] Registering TS with master...
14:06:19.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.627789 25415 heartbeater.cc:507] Master 127.24.53.62:39447 requested a full tablet report, sending...
14:06:19.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:19.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.53.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:19.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.24.53.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:19.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.629753 24851 ts_manager.cc:194] Registered new tserver with Master: 15cfa4072149430492223ea62613b6fd (127.24.53.2:38155)
14:06:19.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.631155 24851 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.24.53.2@KRBTEST.COM'} at 127.24.53.2:42227
14:06:19.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.24.53.3@KRBTEST.COM; defaulting to no policy
14:06:19.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:19.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.24.53.3@KRBTEST.COM" created.
14:06:19.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:06:19.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.24.53.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:19.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.24.53.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab.
14:06:19.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.667276 24788 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:06:19.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:06:19.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-2/wal
14:06:19.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-2/data
14:06:19.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:06:19.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:06:19.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:06:19.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:19.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:19.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-2/logs
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-2/data/info.pb
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.53.3:0
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.53.3
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.53.3
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.53.62:39447,127.24.53.61:38795,127.24.53.60:38575
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.53.3
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:06:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.53.20:36139
14:06:19.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:19.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:19.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:19.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:06:19.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:19 dist-test-slave-sg9n krb5kdc[24800](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649179, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:19.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:19.991642 24915 init.cc:295] Successfully reacquired a new kerberos TGT
14:06:20.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:20.008755 25424 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:06:20.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:20.009284 25424 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:06:20.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:20.009869 25424 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:06:20.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:20.041709 25424 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:06:20.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:20.042769 25424 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.53.3
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:20.081609 25424 tablet_server_runner.cc:78] Tablet server non-default flags:
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.53.20:36139
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-2/data
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-2/wal
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.53.3
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.53.3:0
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-2/data/info.pb
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.53.3
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.53.62:39447,127.24.53.61:38795,127.24.53.60:38575
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.53.3
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-2/logs
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 13:43:23 UTC on 5fd53c4cbb9d
14:06:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7833
14:06:20.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:06:20.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:20.083025 25424 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:06:20.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:20.084054 25424 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:06:20.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:20.098294 25431 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:20.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:20.634589 25415 heartbeater.cc:499] Master 127.24.53.62:39447 was elected leader, sending a full tablet report...
14:06:21.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:20.099848 25432 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:21.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:21.863154 25424 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.763s user 0.592s sys 0.942s
14:06:21.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:21.863549 25424 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.764s user 0.592s sys 0.942s
14:06:21.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:21.501482 25430 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 25424
14:06:21.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:21.866297 25434 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:21.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:21.869480 25433 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1769 milliseconds
14:06:21.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.869493 25424 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:06:21.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.870873 25424 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:06:21.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.873159 25424 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:06:21.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.874578 25424 hybrid_clock.cc:648] HybridClock initialized: now 1756649181874551 us; error 30 us; skew 500 ppm
14:06:21.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.875488 25424 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:06:21.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:21 dist-test-slave-sg9n krb5kdc[24800](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649181, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:21.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.881991 25424 init.cc:377] Logged in from keytab as kudu/127.24.53.3@KRBTEST.COM (short username kudu)
14:06:21.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.887001 25424 webserver.cc:480] Webserver started at http://127.24.53.3:33175/ using document root <none> and password file <none>
14:06:21.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.887979 25424 fs_manager.cc:362] Metadata directory not provided
14:06:21.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.888199 25424 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:06:21.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.888675 25424 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:06:21.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.893257 25424 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-2/data/instance:
14:06:21.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e974c7fb47584732aff774cbb5c86c93"
14:06:21.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:21 on dist-test-slave-sg9n"
14:06:21.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.894356 25424 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-2/wal/instance:
14:06:21.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e974c7fb47584732aff774cbb5c86c93"
14:06:21.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:21 on dist-test-slave-sg9n"
14:06:21.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.901774 25424 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
14:06:21.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.907321 25445 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:06:21.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.908463 25424 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.002s
14:06:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.908835 25424 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-2/data,/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-2/wal
14:06:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e974c7fb47584732aff774cbb5c86c93"
14:06:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:21 on dist-test-slave-sg9n"
14:06:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.909274 25424 fs_report.cc:389] FS layout report
14:06:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:06:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-2/wal
14:06:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-2/wal
14:06:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-2/data/data
14:06:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:06:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:06:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:06:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:06:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:06:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:06:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:06:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:06:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:06:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:06:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:06:21.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.960268 25424 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:06:21.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.970396 25424 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:06:21.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.970901 25424 kserver.cc:163] Server-wide thread pool size limit: 3276
14:06:21.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.974067 25424 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:06:21.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.978914 25424 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:06:21.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.979137 25424 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:06:21.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.979427 25424 ts_tablet_manager.cc:610] Registered 0 tablets
14:06:21.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:21.979638 25424 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:06:22.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.164356 25424 rpc_server.cc:307] RPC server started. Bound to: 127.24.53.3:46431
14:06:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.164714 25558 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.53.3:46431 every 8 connection(s)
14:06:22.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.167572 25424 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-2/data/info.pb
14:06:22.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.175801 24788 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu as pid 25424
14:06:22.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.176256 24788 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/ts-2/wal/instance
14:06:22.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:22 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649181, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.3@KRBTEST.COM for kudu/127.24.53.60@KRBTEST.COM
14:06:22.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:22 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649181, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.3@KRBTEST.COM for kudu/127.24.53.61@KRBTEST.COM
14:06:22.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.293721 25559 heartbeater.cc:344] Connected to a master server at 127.24.53.60:38575
14:06:22.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.294306 25559 heartbeater.cc:461] Registering TS with master...
14:06:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.295850 25559 heartbeater.cc:507] Master 127.24.53.60:38575 requested a full tablet report, sending...
14:06:22.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.301759 25023 ts_manager.cc:194] Registered new tserver with Master: e974c7fb47584732aff774cbb5c86c93 (127.24.53.3:46431)
14:06:22.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:22 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649181, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.3@KRBTEST.COM for kudu/127.24.53.62@KRBTEST.COM
14:06:22.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.315469 25560 heartbeater.cc:344] Connected to a master server at 127.24.53.61:38795
14:06:22.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.315994 25560 heartbeater.cc:461] Registering TS with master...
14:06:22.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.318095 25560 heartbeater.cc:507] Master 127.24.53.61:38795 requested a full tablet report, sending...
14:06:22.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.324160 24936 ts_manager.cc:194] Registered new tserver with Master: e974c7fb47584732aff774cbb5c86c93 (127.24.53.3:46431)
14:06:22.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.326488 25562 heartbeater.cc:344] Connected to a master server at 127.24.53.62:39447
14:06:22.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.327069 25562 heartbeater.cc:461] Registering TS with master...
14:06:22.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.328400 25562 heartbeater.cc:507] Master 127.24.53.62:39447 requested a full tablet report, sending...
14:06:22.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.330575 24851 ts_manager.cc:194] Registered new tserver with Master: e974c7fb47584732aff774cbb5c86c93 (127.24.53.3:46431)
14:06:22.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.332047 24851 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.24.53.3@KRBTEST.COM'} at 127.24.53.3:56225
14:06:22.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.348587 24788 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
14:06:22.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:22 dist-test-slave-sg9n krb5kdc[24800](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649182, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:22.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.353992 25002 init.cc:295] Successfully reacquired a new kerberos TGT
14:06:22.371 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:06:22.372 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
14:06:22.384 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
14:06:22.384 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
14:06:22.388 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.24.53.62"
port: 39447
}
bound_http_address {
host: "127.24.53.62"
port: 40963
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.24.53.61"
port: 38795
}
bound_http_address {
host: "127.24.53.61"
port: 36939
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.24.53.60"
port: 38575
}
bound_http_address {
host: "127.24.53.60"
port: 32857
}
}
}
14:06:22.389 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:06:22.390 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.24.53.1"
port: 34699
}
bound_http_address {
host: "127.24.53.1"
port: 41369
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.24.53.2"
port: 38155
}
bound_http_address {
host: "127.24.53.2"
port: 37793
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.24.53.3"
port: 46431
}
bound_http_address {
host: "127.24.53.3"
port: 33175
}
}
}
14:06:22.390 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:06:22.392 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:06:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:22 dist-test-slave-sg9n krb5kdc[24800](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649182, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:22.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:06:22.435 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:06:22.437 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/krb5cc
14:06:22.444 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:06:22.445 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/krb5cc
14:06:22.446 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:06:22.451 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
14:06:22.453 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:06:22.453 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:06:22.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:22 dist-test-slave-sg9n krb5kdc[24800](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649182, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:22.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:06:22.499 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:06:22.499 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/krb5cc
14:06:22.501 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:06:22.501 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.53.62:39447
14:06:22.502 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:06:22.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.503296 24788 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu with pid 24817
14:06:22.545 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:06:22.546 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.53.60:38575
14:06:22.546 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:06:22.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.546703 24788 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu with pid 24982
14:06:22.572 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:06:22.572 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.24.53.61:38795
14:06:22.573 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:06:22.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.573518 24788 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu with pid 24895
14:06:22.599 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:06:22.599 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.53.62:39447
14:06:22.604 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:06:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.605973 24788 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:06:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:06:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/wal
14:06:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/data
14:06:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:06:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:06:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:06:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/logs
14:06:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/data/info.pb
14:06:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:06:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:06:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:06:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:06:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.53.62:39447
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.53.62
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40963
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.53.62
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.53.20:36139
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.53.62:39447,127.24.53.61:38795,127.24.53.60:38575
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35989/jwks.json
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:06:22.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:06:22.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:22.613045 25411 heartbeater.cc:646] Failed to heartbeat to 127.24.53.60:38575 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.24.53.60:38575: connect: Connection refused (error 111)
14:06:22.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:22.920475 25574 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:06:22.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:22.921056 25574 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:06:22.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:22.921821 25574 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:06:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:22.955152 25574 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:06:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:22.955423 25574 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:06:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:22.955812 25574 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:06:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:22.956149 25574 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:06:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:22.956426 25574 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:06:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:22.956698 25574 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:06:22.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:22.956905 25574 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35989/jwks.json
14:06:22.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.996133 25574 master_runner.cc:386] Master server non-default flags:
14:06:22.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:22.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.53.20:36139
14:06:22.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:22.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:22.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/data
14:06:22.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/wal
14:06:22.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:06:22.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:22.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.53.62:39447,127.24.53.61:38795,127.24.53.60:38575
14:06:22.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab
14:06:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.53.62
14:06:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.53.62:39447
14:06:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:06:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35989/jwks.json
14:06:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:06:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:06:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/data/info.pb
14:06:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.53.62
14:06:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40963
14:06:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:06:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:06:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:06:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/logs
14:06:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:06:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:06:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:06:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:06:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:06:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:06:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 13:43:23 UTC on 5fd53c4cbb9d
14:06:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7833
14:06:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:06:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.997772 25574 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:06:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:22.999173 25574 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:06:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:23.009938 25581 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:23.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:23.310041 25559 heartbeater.cc:646] Failed to heartbeat to 127.24.53.60:38575 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.24.53.60:38575: connect: Connection refused (error 111)
14:06:23.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:23.315934 25266 heartbeater.cc:646] Failed to heartbeat to 127.24.53.62:39447 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.24.53.62:39447: connect: Connection refused (error 111)
14:06:23.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:23.336582 25562 heartbeater.cc:499] Master 127.24.53.62:39447 was elected leader, sending a full tablet report...
14:06:24.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:24.414417 25580 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 25574
14:06:24.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:23.011730 25582 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:24.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:24.846959 25574 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.836s user 0.000s sys 0.002s
14:06:24.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:24.846865 25583 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1834 milliseconds
14:06:24.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:24.848227 25574 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.837s user 0.000s sys 0.003s
14:06:24.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:24.849843 25584 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:24.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:24.850015 25574 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:06:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:24.853736 25574 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:06:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:24.856290 25574 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:06:24.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:24.857740 25574 hybrid_clock.cc:648] HybridClock initialized: now 1756649184857688 us; error 48 us; skew 500 ppm
14:06:24.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:24.858621 25574 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:06:24.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:24 dist-test-slave-sg9n krb5kdc[24800](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649184, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:24.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:24.864814 25574 init.cc:377] Logged in from keytab as kudu/127.24.53.62@KRBTEST.COM (short username kudu)
14:06:24.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:24.870199 25574 webserver.cc:480] Webserver started at http://127.24.53.62:40963/ using document root <none> and password file <none>
14:06:24.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:24.871261 25574 fs_manager.cc:362] Metadata directory not provided
14:06:24.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:24.871515 25574 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:06:24.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:24.879812 25574 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.007s sys 0.001s
14:06:24.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:24.884856 25606 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:06:24.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:24.885969 25574 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
14:06:24.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:24.886327 25574 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/data,/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/wal
14:06:24.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "99305c0f0bbf4ff9934fffd110334b9d"
14:06:24.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:07 on dist-test-slave-sg9n"
14:06:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:24.888499 25574 fs_report.cc:389] FS layout report
14:06:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:06:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/wal
14:06:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/wal
14:06:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/data/data
14:06:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:06:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:06:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:06:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:06:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:06:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:06:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:06:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:06:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:06:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:06:24.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:06:24.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:24.947918 25574 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:06:24.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:24.954063 25574 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:06:24.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:24.954569 25574 kserver.cc:163] Server-wide thread pool size limit: 3276
14:06:25.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.029120 25574 rpc_server.cc:307] RPC server started. Bound to: 127.24.53.62:39447
14:06:25.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.029187 25657 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.53.62:39447 every 8 connection(s)
14:06:25.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.032039 25574 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-0/data/info.pb
14:06:25.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.033111 24788 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu as pid 25574
14:06:25.033 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:06:25.034 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.53.60:38575
14:06:25.034 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:06:25.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.036115 24788 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:06:25.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:06:25.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/wal
14:06:25.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/data
14:06:25.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:06:25.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:06:25.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:06:25.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:25.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/logs
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/data/info.pb
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.53.60:38575
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.53.60
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32857
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.53.60
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:06:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.53.20:36139
14:06:25.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:25.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:25.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:25.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:25.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.53.62:39447,127.24.53.61:38795,127.24.53.60:38575
14:06:25.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:25.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35989/jwks.json
14:06:25.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:06:25.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:06:25.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.043411 25658 sys_catalog.cc:263] Verifying existing consensus state
14:06:25.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.049108 25658 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d: Bootstrap starting.
14:06:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.089529 25658 log.cc:826] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d: Log is configured to *not* fsync() on all Append() calls
14:06:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.105482 25658 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:06:25.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.106352 25658 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d: Bootstrap complete.
14:06:25.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.128698 25658 raft_consensus.cc:357] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } }
14:06:25.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.131117 25658 raft_consensus.cc:738] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 99305c0f0bbf4ff9934fffd110334b9d, State: Initialized, Role: FOLLOWER
14:06:25.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.132050 25658 consensus_queue.cc:260] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } }
14:06:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.135077 25662 sys_catalog.cc:455] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } } }
14:06:25.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.135927 25662 sys_catalog.cc:458] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [sys.catalog]: This master's current role is: FOLLOWER
14:06:25.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.136929 25658 sys_catalog.cc:564] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [sys.catalog]: configured and running, proceeding with master startup.
14:06:25.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.199668 25673 catalog_manager.cc:1261] Loaded cluster ID: b14b064117c2418a8ecf7e52655578ff
14:06:25.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.199959 25673 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d: loading cluster ID for follower catalog manager: success
14:06:25.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:25 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649184, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.62@KRBTEST.COM for kudu/127.24.53.62@KRBTEST.COM
14:06:25.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.230324 25673 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d: acquiring CA information for follower catalog manager: success
14:06:25.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.238168 25673 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:06:25.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.243610 25574 master_runner.cc:186] Error getting master registration for 127.24.53.61:38795: Network error: Client connection negotiation failed: client connection to 127.24.53.61:38795: connect: Connection refused (error 111),
14:06:25.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.261852 25574 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:06:25.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.393929 25266 heartbeater.cc:344] Connected to a master server at 127.24.53.62:39447
14:06:25.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.400913 25623 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "a373d4f104c846e99bd61b11d7185280" instance_seqno: 1756649176913651) as {username='kudu', principal='kudu/127.24.53.1@KRBTEST.COM'} at 127.24.53.1:39705; Asking this server to re-register.
14:06:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.403146 25266 heartbeater.cc:461] Registering TS with master...
14:06:25.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.404520 25562 heartbeater.cc:344] Connected to a master server at 127.24.53.62:39447
14:06:25.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.405031 25562 heartbeater.cc:499] Master 127.24.53.62:39447 was elected leader, sending a full tablet report...
14:06:25.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.407963 25621 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "e974c7fb47584732aff774cbb5c86c93" instance_seqno: 1756649182122225) as {username='kudu', principal='kudu/127.24.53.3@KRBTEST.COM'} at 127.24.53.3:40863; Asking this server to re-register.
14:06:25.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.409689 25622 ts_manager.cc:194] Registered new tserver with Master: a373d4f104c846e99bd61b11d7185280 (127.24.53.1:34699)
14:06:25.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.410167 25562 heartbeater.cc:461] Registering TS with master...
14:06:25.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.412757 25622 ts_manager.cc:194] Registered new tserver with Master: e974c7fb47584732aff774cbb5c86c93 (127.24.53.3:46431)
14:06:25.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:25.498483 25660 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:06:25.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:25.499318 25660 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:06:25.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:25.500445 25660 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:06:25.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:25.558459 25660 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:06:25.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:25.558821 25660 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:06:25.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:25.559324 25660 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:06:25.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:25.559670 25660 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:06:25.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:25.560006 25660 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:06:25.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:25.560328 25660 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:06:25.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:25.560566 25660 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35989/jwks.json
14:06:25.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.614466 25660 master_runner.cc:386] Master server non-default flags:
14:06:25.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.53.20:36139
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/data
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/wal
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.53.62:39447,127.24.53.61:38795,127.24.53.60:38575
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.53.60
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.53.60:38575
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35989/jwks.json
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/data/info.pb
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.53.60
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32857
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:06:25.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:25.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/logs
14:06:25.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:25.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:06:25.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:06:25.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:06:25.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:06:25.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:06:25.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:06:25.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 13:43:23 UTC on 5fd53c4cbb9d
14:06:25.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7833
14:06:25.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:06:25.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.615828 25660 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:06:25.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.617045 25660 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:06:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:25.628698 25690 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.722301 25415 heartbeater.cc:344] Connected to a master server at 127.24.53.62:39447
14:06:25.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.725234 25622 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "15cfa4072149430492223ea62613b6fd" instance_seqno: 1756649179471670) as {username='kudu', principal='kudu/127.24.53.2@KRBTEST.COM'} at 127.24.53.2:37809; Asking this server to re-register.
14:06:25.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.726504 25415 heartbeater.cc:461] Registering TS with master...
14:06:25.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.730010 25622 ts_manager.cc:194] Registered new tserver with Master: 15cfa4072149430492223ea62613b6fd (127.24.53.2:38155)
14:06:25.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:25 dist-test-slave-sg9n krb5kdc[24800](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649185, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:25.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:25.833745 25144 init.cc:295] Successfully reacquired a new kerberos TGT
14:06:26.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:26.504848 25703 raft_consensus.cc:491] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:06:26.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:26.506150 25703 raft_consensus.cc:513] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } }
14:06:26.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:26.560580 25703 leader_election.cc:290] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers ae15ba7f491b4bd19bcb57b0ed510483 (127.24.53.61:38795), 9b08d0ced4ac40308f7432e555465203 (127.24.53.60:38575)
14:06:26.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:26.578392 25610 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.53.60:38575: connect: Connection refused (error 111)
14:06:26.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:26.596386 25610 leader_election.cc:336] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9b08d0ced4ac40308f7432e555465203 (127.24.53.60:38575): Network error: Client connection negotiation failed: client connection to 127.24.53.60:38575: connect: Connection refused (error 111)
14:06:26.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:26.600505 25609 leader_election.cc:336] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer ae15ba7f491b4bd19bcb57b0ed510483 (127.24.53.61:38795): Network error: Client connection negotiation failed: client connection to 127.24.53.61:38795: connect: Connection refused (error 111)
14:06:26.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:26.601438 25609 leader_election.cc:304] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 99305c0f0bbf4ff9934fffd110334b9d; no voters: 9b08d0ced4ac40308f7432e555465203, ae15ba7f491b4bd19bcb57b0ed510483
14:06:26.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:26.604967 25703 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
14:06:26.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:26.961609 24296 debug-util.cc:398] Leaking SignalData structure 0x7b0800086c20 after lost signal to thread 24227
14:06:26.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:26.962791 24296 debug-util.cc:398] Leaking SignalData structure 0x7b08000a6240 after lost signal to thread 24299
14:06:27.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:27.032480 25689 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 25660
14:06:27.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:27.656594 25689 kernel_stack_watchdog.cc:198] Thread 25660 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
14:06:27.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:06:27.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:06:27.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:06:27.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:06:27.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:25.630534 25691 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:27.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:27.662184 25692 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2026 milliseconds
14:06:27.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:27.669373 25693 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:27.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:27.672277 25660 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.041s user 0.000s sys 0.006s
14:06:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:27.672844 25660 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.042s user 0.000s sys 0.006s
14:06:27.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:27.673270 25660 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:06:27.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:27.679327 25660 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:06:27.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:27.694551 25660 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:06:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:27.696460 25660 hybrid_clock.cc:648] HybridClock initialized: now 1756649187696386 us; error 81 us; skew 500 ppm
14:06:27.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:27.697820 25660 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:06:27.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:27 dist-test-slave-sg9n krb5kdc[24800](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649187, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:27.707505 25660 init.cc:377] Logged in from keytab as kudu/127.24.53.60@KRBTEST.COM (short username kudu)
14:06:27.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:27.738274 25660 webserver.cc:480] Webserver started at http://127.24.53.60:32857/ using document root <none> and password file <none>
14:06:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:27.742141 25660 fs_manager.cc:362] Metadata directory not provided
14:06:27.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:27.743975 25660 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:06:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:27.765024 25660 fs_manager.cc:714] Time spent opening directory manager: real 0.014s user 0.008s sys 0.004s
14:06:27.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:27.785383 25722 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:06:27.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:27.789450 25660 fs_manager.cc:730] Time spent opening block manager: real 0.012s user 0.002s sys 0.000s
14:06:27.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:27.790153 25660 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/data,/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/wal
14:06:27.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9b08d0ced4ac40308f7432e555465203"
14:06:27.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:12 on dist-test-slave-sg9n"
14:06:27.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:27.804900 25660 fs_report.cc:389] FS layout report
14:06:27.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:06:27.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/wal
14:06:27.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/wal
14:06:27.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/data/data
14:06:27.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:06:27.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:06:27.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:06:27.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:06:27.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:06:27.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:06:27.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:06:27.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:06:27.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:06:27.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:06:27.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:06:27.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:27.876044 25660 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:06:27.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:27.889585 25660 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:06:27.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:27.890167 25660 kserver.cc:163] Server-wide thread pool size limit: 3276
14:06:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.018245 25660 rpc_server.cc:307] RPC server started. Bound to: 127.24.53.60:38575
14:06:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.018309 25773 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.53.60:38575 every 8 connection(s)
14:06:28.023 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:06:28.024 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.24.53.61:38795
14:06:28.024 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
14:06:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.021379 25660 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-2/data/info.pb
14:06:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.022181 24788 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu as pid 25660
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.026921 24788 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/wal
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/data
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/logs
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/data/info.pb
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.53.61:38795
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.53.61
14:06:28.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36939
14:06:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab
14:06:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.53.61
14:06:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:06:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:06:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:06:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.53.20:36139
14:06:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.53.62:39447,127.24.53.61:38795,127.24.53.60:38575
14:06:28.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:28.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35989/jwks.json
14:06:28.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:06:28.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:06:28.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.075814 25774 sys_catalog.cc:263] Verifying existing consensus state
14:06:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.085433 25774 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203: Bootstrap starting.
14:06:28.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.138195 25774 log.cc:826] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203: Log is configured to *not* fsync() on all Append() calls
14:06:28.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.207458 25774 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:06:28.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.208472 25774 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203: Bootstrap complete.
14:06:28.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.269258 25774 raft_consensus.cc:357] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } }
14:06:28.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.273440 25774 raft_consensus.cc:738] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9b08d0ced4ac40308f7432e555465203, State: Initialized, Role: FOLLOWER
14:06:28.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.274849 25774 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } }
14:06:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.291746 25780 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } } }
14:06:28.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.292690 25780 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [sys.catalog]: This master's current role is: FOLLOWER
14:06:28.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.294641 25774 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [sys.catalog]: configured and running, proceeding with master startup.
14:06:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.388015 25791 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:06:28.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.404307 25792 raft_consensus.cc:491] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:06:28.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.405143 25792 raft_consensus.cc:513] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } }
14:06:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.409247 25792 leader_election.cc:290] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers ae15ba7f491b4bd19bcb57b0ed510483 (127.24.53.61:38795), 9b08d0ced4ac40308f7432e555465203 (127.24.53.60:38575)
14:06:28.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:28 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649187, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.60@KRBTEST.COM for kudu/127.24.53.62@KRBTEST.COM
14:06:28.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:28.473371 25609 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.24.53.61:38795, user_credentials={real_user=slave}} blocked reactor thread for 50902us
14:06:28.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:28.477880 25609 leader_election.cc:336] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer ae15ba7f491b4bd19bcb57b0ed510483 (127.24.53.61:38795): Network error: Client connection negotiation failed: client connection to 127.24.53.61:38795: connect: Connection refused (error 111)
14:06:28.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:28 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649184, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.62@KRBTEST.COM for kudu/127.24.53.60@KRBTEST.COM
14:06:28.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:28 dist-test-slave-sg9n krb5kdc[24800](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649185, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.1@KRBTEST.COM for kudu/127.24.53.60@KRBTEST.COM
14:06:28.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.547874 25261 heartbeater.cc:344] Connected to a master server at 127.24.53.60:38575
14:06:28.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.525959 25749 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "99305c0f0bbf4ff9934fffd110334b9d" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "9b08d0ced4ac40308f7432e555465203" is_pre_election: true
14:06:28.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.568531 25749 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 99305c0f0bbf4ff9934fffd110334b9d in term 1.
14:06:28.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.571203 25610 leader_election.cc:304] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 99305c0f0bbf4ff9934fffd110334b9d, 9b08d0ced4ac40308f7432e555465203; no voters: ae15ba7f491b4bd19bcb57b0ed510483
14:06:28.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.572981 25739 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "a373d4f104c846e99bd61b11d7185280" instance_seqno: 1756649176913651) as {username='kudu', principal='kudu/127.24.53.1@KRBTEST.COM'} at 127.24.53.1:57985; Asking this server to re-register.
14:06:28.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.574193 25792 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 1 FOLLOWER]: Leader pre-election won for term 2
14:06:28.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.488636 25660 master_runner.cc:186] Error getting master registration for 127.24.53.61:38795: Network error: Client connection negotiation failed: client connection to 127.24.53.61:38795: connect: Connection refused (error 111),
14:06:28.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.574553 25792 raft_consensus.cc:491] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:06:28.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.574893 25792 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 1 FOLLOWER]: Advancing to term 2
14:06:28.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.575515 25559 heartbeater.cc:344] Connected to a master server at 127.24.53.60:38575
14:06:28.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.574833 25660 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:06:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.577795 25261 heartbeater.cc:461] Registering TS with master...
14:06:28.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.578166 25739 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "e974c7fb47584732aff774cbb5c86c93" instance_seqno: 1756649182122225) as {username='kudu', principal='kudu/127.24.53.3@KRBTEST.COM'} at 127.24.53.3:36201; Asking this server to re-register.
14:06:28.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.579325 25559 heartbeater.cc:461] Registering TS with master...
14:06:28.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.582935 25738 ts_manager.cc:194] Registered new tserver with Master: e974c7fb47584732aff774cbb5c86c93 (127.24.53.3:46431)
14:06:28.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.582595 25739 ts_manager.cc:194] Registered new tserver with Master: a373d4f104c846e99bd61b11d7185280 (127.24.53.1:34699)
14:06:28.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.587129 25792 raft_consensus.cc:513] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } }
14:06:28.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:28.593068 25609 leader_election.cc:336] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer ae15ba7f491b4bd19bcb57b0ed510483 (127.24.53.61:38795): Network error: Client connection negotiation failed: client connection to 127.24.53.61:38795: connect: Connection refused (error 111)
14:06:28.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.594547 25792 leader_election.cc:290] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [CANDIDATE]: Term 2 election: Requested vote from peers ae15ba7f491b4bd19bcb57b0ed510483 (127.24.53.61:38795), 9b08d0ced4ac40308f7432e555465203 (127.24.53.60:38575)
14:06:28.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.595273 25749 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "99305c0f0bbf4ff9934fffd110334b9d" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "9b08d0ced4ac40308f7432e555465203"
14:06:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.595865 25749 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [term 1 FOLLOWER]: Advancing to term 2
14:06:28.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.605258 25749 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 99305c0f0bbf4ff9934fffd110334b9d in term 2.
14:06:28.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.606500 25610 leader_election.cc:304] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 99305c0f0bbf4ff9934fffd110334b9d, 9b08d0ced4ac40308f7432e555465203; no voters: ae15ba7f491b4bd19bcb57b0ed510483
14:06:28.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.607568 25792 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 2 FOLLOWER]: Leader election won for term 2
14:06:28.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.620622 25792 raft_consensus.cc:695] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [term 2 LEADER]: Becoming Leader. State: Replica: 99305c0f0bbf4ff9934fffd110334b9d, State: Running, Role: LEADER
14:06:28.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.621912 25792 consensus_queue.cc:237] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } }
14:06:28.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.635522 25806 sys_catalog.cc:455] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 99305c0f0bbf4ff9934fffd110334b9d. Latest consensus state: current_term: 2 leader_uuid: "99305c0f0bbf4ff9934fffd110334b9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } } }
14:06:28.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.636529 25806 sys_catalog.cc:458] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [sys.catalog]: This master's current role is: LEADER
14:06:28.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.644920 25808 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:06:28.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.654491 25808 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:06:28.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.658874 25808 catalog_manager.cc:1261] Loaded cluster ID: b14b064117c2418a8ecf7e52655578ff
14:06:28.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.659375 25808 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:06:28.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.665295 25808 catalog_manager.cc:1506] Loading token signing keys...
14:06:28.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.681077 25808 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d: Loaded TSK: 0
14:06:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.688365 25808 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:06:28.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:28.787717 25775 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:06:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:28.789208 25775 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:06:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:28.791018 25775 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:06:28.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.827065 25411 heartbeater.cc:344] Connected to a master server at 127.24.53.60:38575
14:06:28.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.831247 25739 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "15cfa4072149430492223ea62613b6fd" instance_seqno: 1756649179471670) as {username='kudu', principal='kudu/127.24.53.2@KRBTEST.COM'} at 127.24.53.2:60741; Asking this server to re-register.
14:06:28.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.833021 25411 heartbeater.cc:461] Registering TS with master...
14:06:28.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.842036 25739 ts_manager.cc:194] Registered new tserver with Master: 15cfa4072149430492223ea62613b6fd (127.24.53.2:38155)
14:06:28.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:28.850942 25775 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:06:28.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:28.851277 25775 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:06:28.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:28.851861 25775 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:06:28.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:28.852228 25775 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:06:28.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:28.852557 25775 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:06:28.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:28.852882 25775 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:06:28.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:28.853155 25775 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35989/jwks.json
14:06:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.925577 25775 master_runner.cc:386] Master server non-default flags:
14:06:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.53.20:36139
14:06:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/data
14:06:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/wal
14:06:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:06:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:28.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.53.62:39447,127.24.53.61:38795,127.24.53.60:38575
14:06:28.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/krb5kdc/kudu.keytab
14:06:28.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.24.53.61
14:06:28.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:28.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:28.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:28.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.53.61:38795
14:06:28.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:28.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:06:28.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:28.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35989/jwks.json
14:06:28.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:06:28.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:06:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:06:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/data/info.pb
14:06:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.53.61
14:06:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36939
14:06:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:06:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:06:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:06:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/logs
14:06:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:06:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:06:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:06:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:06:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:06:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:06:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 13:43:23 UTC on 5fd53c4cbb9d
14:06:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7833
14:06:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:06:28.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.927331 25775 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:06:28.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:28.929518 25775 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:06:28.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:28.965896 25816 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:29.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:29.041586 25749 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [term 2 FOLLOWER]: Refusing update from remote peer 99305c0f0bbf4ff9934fffd110334b9d: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
14:06:29.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:29.043457 25806 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
14:06:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:29.073448 25609 consensus_peers.cc:489] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d -> Peer ae15ba7f491b4bd19bcb57b0ed510483 (127.24.53.61:38795): Couldn't send request to peer ae15ba7f491b4bd19bcb57b0ed510483. Status: Network error: Client connection negotiation failed: client connection to 127.24.53.61:38795: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:06:29.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:29.105732 25806 sys_catalog.cc:455] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "99305c0f0bbf4ff9934fffd110334b9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } } }
14:06:29.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:29.106472 25806 sys_catalog.cc:458] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d [sys.catalog]: This master's current role is: LEADER
14:06:29.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:29.117002 25826 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "99305c0f0bbf4ff9934fffd110334b9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } } }
14:06:29.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:29.117662 25826 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [sys.catalog]: This master's current role is: FOLLOWER
14:06:29.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:29.109113 25822 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 99305c0f0bbf4ff9934fffd110334b9d. Latest consensus state: current_term: 2 leader_uuid: "99305c0f0bbf4ff9934fffd110334b9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } } }
14:06:29.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:29.118737 25822 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203 [sys.catalog]: This master's current role is: FOLLOWER
14:06:29.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:29 dist-test-slave-sg9n krb5kdc[24800](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649189, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:29.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:29.219020 25293 init.cc:295] Successfully reacquired a new kerberos TGT
14:06:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:29.413137 25791 catalog_manager.cc:1261] Loaded cluster ID: b14b064117c2418a8ecf7e52655578ff
14:06:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:29.413535 25791 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203: loading cluster ID for follower catalog manager: success
14:06:29.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:29.444463 25791 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203: acquiring CA information for follower catalog manager: success
14:06:29.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:29.469549 25791 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 9b08d0ced4ac40308f7432e555465203: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:06:29.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:29.747735 25415 heartbeater.cc:499] Master 127.24.53.62:39447 was elected leader, sending a full tablet report...
14:06:30.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:30.269778 25574 master_runner.cc:186] Error getting master registration for 127.24.53.61:38795: Network error: Client connection negotiation failed: client connection to 127.24.53.61:38795: connect: Connection refused (error 111),
14:06:30.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:30.270471 25574 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:06:30.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:30.441681 25266 heartbeater.cc:499] Master 127.24.53.62:39447 was elected leader, sending a full tablet report...
14:06:30.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:30.443198 25562 heartbeater.cc:499] Master 127.24.53.62:39447 was elected leader, sending a full tablet report...
14:06:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:31.381323 25609 consensus_peers.cc:489] T 00000000000000000000000000000000 P 99305c0f0bbf4ff9934fffd110334b9d -> Peer ae15ba7f491b4bd19bcb57b0ed510483 (127.24.53.61:38795): Couldn't send request to peer ae15ba7f491b4bd19bcb57b0ed510483. Status: Network error: Client connection negotiation failed: client connection to 127.24.53.61:38795: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
14:06:31.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:31.765995 25609 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.53.61:38795: connect: Connection refused (error 111) [suppressed 9 similar messages]
14:06:31.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:30.357118 25815 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 25775
14:06:31.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:31.773298 25815 kernel_stack_watchdog.cc:198] Thread 25775 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 383ms:
14:06:31.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:06:31.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:06:31.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:06:31.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:06:31.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:28.968492 25817 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:31.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:31.774938 25818 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2804 milliseconds
14:06:31.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:31.779142 25775 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.807s user 0.000s sys 0.006s
14:06:31.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:31.779537 25775 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.808s user 0.000s sys 0.006s
14:06:31.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:31.781026 25820 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:06:31.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:31.781064 25775 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:06:31.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:31.783063 25775 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:06:31.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:31.786545 25775 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:06:31.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:31.787993 25775 hybrid_clock.cc:648] HybridClock initialized: now 1756649191787900 us; error 107 us; skew 500 ppm
14:06:31.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:31.788892 25775 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:06:31.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:31 dist-test-slave-sg9n krb5kdc[24800](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649191, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:31.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:31.795120 25775 init.cc:377] Logged in from keytab as kudu/127.24.53.61@KRBTEST.COM (short username kudu)
14:06:31.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:31.801520 25775 webserver.cc:480] Webserver started at http://127.24.53.61:36939/ using document root <none> and password file <none>
14:06:31.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:31.802634 25775 fs_manager.cc:362] Metadata directory not provided
14:06:31.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:31.802927 25775 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:06:31.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:31.811501 25775 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.004s sys 0.004s
14:06:31.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:31.816694 25845 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:06:31.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:31.817967 25775 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
14:06:31.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:31.818372 25775 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/data,/tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/wal
14:06:31.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae15ba7f491b4bd19bcb57b0ed510483"
14:06:31.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 14:06:09 on dist-test-slave-sg9n"
14:06:31.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:31.820477 25775 fs_report.cc:389] FS layout report
14:06:31.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:06:31.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/wal
14:06:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/wal
14:06:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/data/data
14:06:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:06:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:06:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:06:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:06:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:06:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:06:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:06:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:06:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:06:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:06:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:06:31.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 14:06:31 dist-test-slave-sg9n krb5kdc[24800](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756649191, etypes {rep=17 tkt=17 ses=17}, kudu/127.24.53.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:06:31.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:31.887413 25441 init.cc:295] Successfully reacquired a new kerberos TGT
14:06:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:31.893999 25775 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:06:31.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:31.901461 25775 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:06:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:31.902017 25775 kserver.cc:163] Server-wide thread pool size limit: 3276
14:06:31.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:31.990792 25775 rpc_server.cc:307] RPC server started. Bound to: 127.24.53.61:38795
14:06:31.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:31.990900 25897 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.24.53.61:38795 every 8 connection(s)
14:06:31.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:31.994848 25775 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskqLh0aI/test-tmp/mini-kudu-cluster2411828466651831555/master-1/data/info.pb
14:06:32.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.000844 24788 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu as pid 25775
14:06:32.001 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:06:32.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.010493 25898 sys_catalog.cc:263] Verifying existing consensus state
14:06:32.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.016778 25898 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483: Bootstrap starting.
14:06:32.063 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.24.53.62:39447(127.24.53.62:39447)] connecting to peer
14:06:32.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.089759 25898 log.cc:826] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483: Log is configured to *not* fsync() on all Append() calls
14:06:32.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.117534 25898 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:06:32.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.118566 25898 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483: Bootstrap complete.
14:06:32.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.156193 25898 raft_consensus.cc:357] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } }
14:06:32.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.160550 25898 raft_consensus.cc:738] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: ae15ba7f491b4bd19bcb57b0ed510483, State: Initialized, Role: FOLLOWER
14:06:32.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.161718 25898 consensus_queue.cc:260] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } }
14:06:32.167 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.24.53.60:38575(127.24.53.60:38575)] connecting to peer
14:06:32.179 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.24.53.61:38795(127.24.53.61:38795)] connecting to peer
14:06:32.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.219949 25904 sys_catalog.cc:455] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } } }
14:06:32.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.222463 25904 sys_catalog.cc:458] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [sys.catalog]: This master's current role is: FOLLOWER
14:06:32.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.221988 25898 sys_catalog.cc:564] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [sys.catalog]: configured and running, proceeding with master startup.
14:06:32.244 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.24.53.60:38575(127.24.53.60:38575)] Successfully connected to peer
14:06:32.265 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.24.53.62:39447(127.24.53.62:39447)] Successfully connected to peer
14:06:32.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.305078 25919 catalog_manager.cc:1261] Loaded cluster ID: b14b064117c2418a8ecf7e52655578ff
14:06:32.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.305423 25919 catalog_manager.cc:1554] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483: loading cluster ID for follower catalog manager: success
14:06:32.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.315471 25919 catalog_manager.cc:1576] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483: acquiring CA information for follower catalog manager: success
14:06:32.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.321906 25919 catalog_manager.cc:1604] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:06:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.415632 25873 raft_consensus.cc:3058] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [term 1 FOLLOWER]: Advancing to term 2
14:06:32.429 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.24.53.61:38795(127.24.53.61:38795)] Successfully connected to peer
14:06:32.460 [WARN - kudu-nio-1] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
14:06:32.472 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.53.62:39447(127.24.53.62:39447)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:06:32.474 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.53.62:39447: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:06:32.476 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.24.53.60:38575(127.24.53.60:38575)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:06:32.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:32.477577 25910 negotiation.cc:337] Failed RPC negotiation. Trace:
14:06:32.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.251356 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:43716 (local address 127.24.53.62:39447)
14:06:32.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.280538 (+ 29182us) server_negotiation.cc:206] Beginning negotiation
14:06:32.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.280578 (+ 40us) server_negotiation.cc:399] Waiting for connection header
14:06:32.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.363108 (+ 82530us) server_negotiation.cc:407] Connection header received
14:06:32.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.429013 (+ 65905us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:06:32.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.429049 (+ 36us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:06:32.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.429301 (+ 252us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:06:32.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.430008 (+ 707us) server_negotiation.cc:226] Negotiated authn=SASL
14:06:32.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.24.53.60:38575: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:06:32.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.475494 (+ 45486us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:43716: BlockingRecv error: recv got EOF from 127.0.0.1:43716 (error 108)
14:06:32.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":27680,"thread_start_us":31635,"threads_started":1}
14:06:32.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.477661 25904 sys_catalog.cc:455] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 99305c0f0bbf4ff9934fffd110334b9d. Latest consensus state: current_term: 2 leader_uuid: "99305c0f0bbf4ff9934fffd110334b9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } } }
14:06:32.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.478405 25904 sys_catalog.cc:458] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [sys.catalog]: This master's current role is: FOLLOWER
14:06:32.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.480810 25929 sys_catalog.cc:455] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "99305c0f0bbf4ff9934fffd110334b9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "99305c0f0bbf4ff9934fffd110334b9d" member_type: VOTER last_known_addr { host: "127.24.53.62" port: 39447 } } peers { permanent_uuid: "ae15ba7f491b4bd19bcb57b0ed510483" member_type: VOTER last_known_addr { host: "127.24.53.61" port: 38795 } } peers { permanent_uuid: "9b08d0ced4ac40308f7432e555465203" member_type: VOTER last_known_addr { host: "127.24.53.60" port: 38575 } } }
14:06:32.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.481626 25929 sys_catalog.cc:458] T 00000000000000000000000000000000 P ae15ba7f491b4bd19bcb57b0ed510483 [sys.catalog]: This master's current role is: FOLLOWER
14:06:32.482 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.24.53.60:38575(127.24.53.60:38575)] handling channelInactive
14:06:32.483 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.24.53.61:38795(127.24.53.61:38795)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:06:32.483 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.24.53.61:38795: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:06:32.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:32.484498 25908 negotiation.cc:337] Failed RPC negotiation. Trace:
14:06:32.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.237042 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:53158 (local address 127.24.53.60:38575)
14:06:32.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.282132 (+ 45090us) server_negotiation.cc:206] Beginning negotiation
14:06:32.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.282170 (+ 38us) server_negotiation.cc:399] Waiting for connection header
14:06:32.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.363279 (+ 81109us) server_negotiation.cc:407] Connection header received
14:06:32.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.435941 (+ 72662us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:06:32.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.435970 (+ 29us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:06:32.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.436214 (+ 244us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:06:32.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.436386 (+ 172us) server_negotiation.cc:226] Negotiated authn=SASL
14:06:32.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.482744 (+ 46358us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:53158: BlockingRecv error: recv got EOF from 127.0.0.1:53158 (error 108)
14:06:32.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":44220,"spinlock_wait_cycles":60544,"thread_start_us":45702,"threads_started":1}
14:06:32.490 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=483), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.53.62:39447, [155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.53.60:38575, [167ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.53.61:38795, [461ms] Sub RPC ConnectToMaster: received response from server master-127.24.53.62:39447: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [464ms] Sub RPC ConnectToMaster: received response from server master-127.24.53.60:38575: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [471ms] Sub RPC ConnectToMaster: received response from server master-127.24.53.61:38795: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.24.53.62:39447,127.24.53.60:38575,127.24.53.61:38795). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) [main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) [main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) [async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) [main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) [main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) [main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) [netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.24.53.62:39447,127.24.53.60:38575,127.24.53.61:38795). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
14:06:32.496 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=489), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.53.62:39447, [155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.53.60:38575, [167ms] Sub RPC ConnectToMaster: sending RPC to server master-127.24.53.61:38795, [461ms] Sub RPC ConnectToMaster: received response from server master-127.24.53.62:39447: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [464ms] Sub RPC ConnectToMaster: received response from server master-127.24.53.60:38575: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [471ms] Sub RPC ConnectToMaster: received response from server master-127.24.53.61:38795: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) with no deferred attached!
14:06:32.497 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.24.53.62:39447(127.24.53.62:39447)] handling channelInactive
14:06:32.497 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.24.53.61:38795(127.24.53.61:38795)] handling channelInactive
14:06:32.503 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.24.53.61:38795(127.24.53.61:38795)] disconnecting while in state TERMINATED
14:06:32.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:32.500348 25912 negotiation.cc:337] Failed RPC negotiation. Trace:
14:06:32.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.263160 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:40596 (local address 127.24.53.61:38795)
14:06:32.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.278459 (+ 15299us) server_negotiation.cc:206] Beginning negotiation
14:06:32.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.278496 (+ 37us) server_negotiation.cc:399] Waiting for connection header
14:06:32.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.429924 (+151428us) server_negotiation.cc:407] Connection header received
14:06:32.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.430986 (+ 1062us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:06:32.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.431033 (+ 47us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:06:32.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.431427 (+ 394us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:06:32.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.431675 (+ 248us) server_negotiation.cc:226] Negotiated authn=SASL
14:06:32.504 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.24.53.60:38575(127.24.53.60:38575)] disconnecting while in state TERMINATED
14:06:32.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 14:06:32.498325 (+ 66650us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:40596: BlockingRecv error: recv got EOF from 127.0.0.1:40596 (error 108)
14:06:32.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":14553,"thread_start_us":14229,"threads_started":1}
14:06:32.508 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.24.53.62:39447(127.24.53.62:39447)] disconnecting while in state TERMINATED
14:06:32.511 [DEBUG - Time-limited test] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:06:32.516 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:06:32.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.528898 24788 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu with pid 25125
14:06:32.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.572649 24788 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu with pid 25275
14:06:32.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.621483 25560 heartbeater.cc:344] Connected to a master server at 127.24.53.61:38795
14:06:32.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.623965 24788 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu with pid 25424
14:06:32.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.629244 25863 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "e974c7fb47584732aff774cbb5c86c93" instance_seqno: 1756649182122225) as {username='kudu', principal='kudu/127.24.53.3@KRBTEST.COM'} at 127.24.53.3:47553; Asking this server to re-register.
14:06:32.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 14:06:32.658239 25850 connection.cc:537] server connection from 127.24.53.3:47553 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:06:32.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.659849 24788 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu with pid 25574
14:06:32.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.718891 24788 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu with pid 25775
14:06:32.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:32.738569 24788 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu with pid 25660
14:06:33.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-08-31T14:06:33Z chronyd exiting
14:06:35.008 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:06:35.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:35.009479 24065 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu with pid 24348
14:06:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:35.034889 24065 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu with pid 24505
14:06:35.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:35.062088 24065 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu with pid 24647
14:06:35.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:35.087797 24065 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu with pid 24077
14:06:35.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:35.117413 24065 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu with pid 24147
14:06:35.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 14:06:35.148048 24065 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskqLh0aI/build/tsan/bin/kudu with pid 24222
14:06:36.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-08-31T14:06:36Z chronyd exiting
14:06:37.331 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.24.53.62:39447,127.24.53.60:38575,127.24.53.61:38795). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.24.53.62:39447,127.24.53.60:38575,127.24.53.61:38795). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
14:06:37.755 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
14:06:37.756 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.24.53.62:39447,127.24.53.60:38575,127.24.53.61:38795). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.24.53.62:39447,127.24.53.60:38575,127.24.53.61:38795). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more