09:03:01.950 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
09:03:01.964 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskQWUX02/test-tmp
09:03:01.971 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
09:03:01.974 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskQWUX02/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
09:03:02.159 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
09:03:02.357 [DEBUG - main] (MiniKuduCluster.java:195) Response:
09:03:02.358 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
09:03:02.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:02.363816 795 mini_oidc.cc:150] Starting JWKS server
09:03:02.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:02.367973 795 webserver.cc:577] Webserver started at http://127.0.0.1:39947/ using document root <none> and password file <none>
09:03:02.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:02.368539 795 mini_oidc.cc:186] Starting OIDC Discovery server
09:03:02.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:02.370636 795 webserver.cc:577] Webserver started at http://127.0.0.1:46237/ using document root <none> and password file <none>
09:03:02.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-16T09:03:02Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
09:03:02.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-16T09:03:02Z Disabled control of system clock
09:03:02.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:02.434048 795 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:02.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:02.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-0/wal
09:03:02.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-0/data
09:03:02.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:02.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:02.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:02.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:02.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:02.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:02.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-0/logs
09:03:02.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-0/data/info.pb
09:03:02.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:02.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:02.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:02.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:02.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:02.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:02.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:02.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:02.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:03:02.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:02.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:02.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:02.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.198.254:43159
09:03:02.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.198.254
09:03:02.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:02.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.198.212:46237
09:03:02.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:02.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:02.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:02.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.198.254:43159,127.0.198.253:44451,127.0.198.252:35341
09:03:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39947/jwks.json with env {}
09:03:02.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:02.743746 807 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:02.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:02.744305 807 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:02.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:02.744987 807 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:02.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:02.778963 807 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
09:03:02.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:02.779253 807 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:02.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:02.779453 807 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
09:03:02.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:02.779672 807 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
09:03:02.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:02.779994 807 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
09:03:02.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:02.780257 807 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39947/jwks.json
09:03:02.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:02.818204 807 master_runner.cc:387] Master server non-default flags:
09:03:02.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-0/logs
09:03:02.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:02.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:02.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:02.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.198.212:46237
09:03:02.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-0/data
09:03:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-0/wal
09:03:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.198.254:43159,127.0.198.253:44451,127.0.198.252:35341
09:03:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.198.254:43159
09:03:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39947/jwks.json
09:03:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-0/data/info.pb
09:03:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.198.254
09:03:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:02.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:02.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:02.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:02.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:03:02.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:03:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Nov 2026 08:43:21 UTC on e7f111948823
09:03:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14810
09:03:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:02.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:02.819571 807 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:02.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:02.821694 807 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:02.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:02.836978 813 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:04.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:02.836985 814 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:04.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:04.051359 807 thread.cc:642] GCE (cloud detector) Time spent creating pthread: real 1.216s user 0.442s sys 0.772s
09:03:04.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:04.051759 807 thread.cc:609] GCE (cloud detector) Time spent starting thread: real 1.217s user 0.442s sys 0.772s
09:03:04.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.054725 807 server_base.cc:1061] running on GCE node
09:03:04.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:04.055236 818 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:04.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.056562 807 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:04.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.059165 807 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:04.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.060580 807 hybrid_clock.cc:648] HybridClock initialized: now 1794819784060532 us; error 42 us; skew 500 ppm
09:03:04.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.061806 807 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:04.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.070407 807 webserver.cc:577] Webserver started at http://127.0.198.254:46077/ using document root <none> and password file <none>
09:03:04.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.071455 807 fs_manager.cc:362] Metadata directory not provided
09:03:04.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.071645 807 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:04.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.072140 807 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:03:04.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.076520 807 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-0/data/instance:
09:03:04.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "defc3d7aa00d416b9b764c0b6665053c"
09:03:04.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:04 on dist-test-slave-sqmp"
09:03:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.077907 807 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-0/wal/instance:
09:03:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "defc3d7aa00d416b9b764c0b6665053c"
09:03:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:04 on dist-test-slave-sqmp"
09:03:04.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.086385 807 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.000s
09:03:04.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.092339 823 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:03:04.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.093776 807 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.001s
09:03:04.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.094159 807 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-0/data,/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-0/wal
09:03:04.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "defc3d7aa00d416b9b764c0b6665053c"
09:03:04.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:04 on dist-test-slave-sqmp"
09:03:04.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.094594 807 fs_report.cc:389] FS layout report
09:03:04.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:04.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-0/wal
09:03:04.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-0/wal
09:03:04.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-0/data/data
09:03:04.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:04.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:04.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:04.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:04.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:04.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:04.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:04.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.160838 807 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:03:04.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.162552 807 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:04.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.163060 807 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:04.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.249332 807 rpc_server.cc:307] RPC server started. Bound to: 127.0.198.254:43159
09:03:04.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.249460 874 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.198.254:43159 every 8 connection(s)
09:03:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.252492 807 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-0/data/info.pb
09:03:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.257957 795 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu as pid 807
09:03:04.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.258420 795 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-0/wal/instance
09:03:04.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.258831 875 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:04.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.262092 795 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:04.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:04.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-1/wal
09:03:04.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-1/data
09:03:04.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:04.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:04.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:04.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:04.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:04.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:04.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-1/logs
09:03:04.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-1/data/info.pb
09:03:04.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:04.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:04.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:04.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:04.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:04.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:04.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:04.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:04.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:03:04.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:04.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:04.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:04.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.198.253:44451
09:03:04.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.198.253
09:03:04.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:04.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.198.212:46237
09:03:04.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:04.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:04.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:04.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:04.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.198.254:43159,127.0.198.253:44451,127.0.198.252:35341
09:03:04.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:04.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39947/jwks.json with env {}
09:03:04.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.277001 875 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.198.254" port: 43159 } has no permanent_uuid. Determining permanent_uuid...
09:03:04.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.308763 875 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } has no permanent_uuid. Determining permanent_uuid...
09:03:04.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:04.311523 826 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.198.253:44451: connect: Connection refused (error 111)
09:03:04.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:04.315923 875 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.198.253:44451: Network error: Client connection negotiation failed: client connection to 127.0.198.253:44451: connect: Connection refused (error 111)
09:03:04.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.365499 875 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } attempt: 1
09:03:04.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:04.369881 875 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.198.253:44451: Network error: Client connection negotiation failed: client connection to 127.0.198.253:44451: connect: Connection refused (error 111)
09:03:04.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.438349 875 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } attempt: 2
09:03:04.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:04.443411 875 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.198.253:44451: Network error: Client connection negotiation failed: client connection to 127.0.198.253:44451: connect: Connection refused (error 111)
09:03:04.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.534847 875 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } attempt: 3
09:03:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:04.539557 875 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.198.253:44451: Network error: Client connection negotiation failed: client connection to 127.0.198.253:44451: connect: Connection refused (error 111)
09:03:04.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:04.584967 877 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:04.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:04.585866 877 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:04.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:04.586453 877 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:04.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:04.618978 877 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
09:03:04.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:04.619280 877 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:04.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:04.619575 877 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
09:03:04.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:04.619783 877 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
09:03:04.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:04.620095 877 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
09:03:04.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:04.620360 877 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39947/jwks.json
09:03:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.671417 877 master_runner.cc:387] Master server non-default flags:
09:03:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-1/logs
09:03:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.198.212:46237
09:03:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-1/data
09:03:04.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-1/wal
09:03:04.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:04.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.198.254:43159,127.0.198.253:44451,127.0.198.252:35341
09:03:04.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:04.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:04.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:04.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.198.253:44451
09:03:04.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:04.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:04.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:04.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39947/jwks.json
09:03:04.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:04.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:04.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-1/data/info.pb
09:03:04.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.198.253
09:03:04.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:04.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:04.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:04.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:04.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:04.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:04.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:03:04.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:03:04.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:04.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:04.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:04.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Nov 2026 08:43:21 UTC on e7f111948823
09:03:04.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14810
09:03:04.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:04.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.672813 877 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:04.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.674612 877 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:04.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.682952 875 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } attempt: 4
09:03:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:04.687420 886 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:04.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:04.689317 875 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.198.253:44451: Network error: Client connection negotiation failed: client connection to 127.0.198.253:44451: connect: Connection refused (error 111)
09:03:04.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.988982 875 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } attempt: 5
09:03:04.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:04.993141 875 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.198.253:44451: Network error: Client connection negotiation failed: client connection to 127.0.198.253:44451: connect: Connection refused (error 111)
09:03:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:05.540802 875 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } attempt: 6
09:03:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:05.550169 875 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.198.253:44451: Network error: Client connection negotiation failed: client connection to 127.0.198.253:44451: connect: Connection refused (error 111)
09:03:05.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:04.691614 887 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:05.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:04.693100 877 server_base.cc:1061] running on GCE node
09:03:05.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:04.692226 889 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:05.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:05.887122 877 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:05.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:05.889771 877 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:05.891106 877 hybrid_clock.cc:648] HybridClock initialized: now 1794819785891070 us; error 78 us; skew 500 ppm
09:03:05.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:05.891919 877 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:05.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:05.897746 877 webserver.cc:577] Webserver started at http://127.0.198.253:43805/ using document root <none> and password file <none>
09:03:05.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:05.898622 877 fs_manager.cc:362] Metadata directory not provided
09:03:05.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:05.898805 877 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:05.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:05.899183 877 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:03:05.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:05.903324 877 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-1/data/instance:
09:03:05.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae11cc96c8a140dda8a76dbcf182c925"
09:03:05.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:05 on dist-test-slave-sqmp"
09:03:05.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:05.904538 877 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-1/wal/instance:
09:03:05.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae11cc96c8a140dda8a76dbcf182c925"
09:03:05.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:05 on dist-test-slave-sqmp"
09:03:05.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:05.911464 877 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.009s sys 0.000s
09:03:05.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:05.917603 898 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:03:05.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:05.918761 877 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
09:03:05.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:05.919031 877 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-1/data,/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-1/wal
09:03:05.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ae11cc96c8a140dda8a76dbcf182c925"
09:03:05.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:05 on dist-test-slave-sqmp"
09:03:05.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:05.919313 877 fs_report.cc:389] FS layout report
09:03:05.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:05.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-1/wal
09:03:05.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-1/wal
09:03:05.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-1/data/data
09:03:05.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:05.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:05.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:05.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:05.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:05.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:05.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:05.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:05.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:05.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:05.972297 877 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:03:05.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:05.973827 877 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:05.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:05.974215 877 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:06.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:06.043192 877 rpc_server.cc:307] RPC server started. Bound to: 127.0.198.253:44451
09:03:06.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:06.043268 949 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.198.253:44451 every 8 connection(s)
09:03:06.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:06.046554 877 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-1/data/info.pb
09:03:06.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:06.053900 950 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:06.054996 795 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu as pid 877
09:03:06.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:06.055465 795 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-1/wal/instance
09:03:06.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:06.057957 795 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:06.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:06.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-2/wal
09:03:06.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-2/data
09:03:06.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:06.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:06.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:06.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:06.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:06.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:06.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-2/logs
09:03:06.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-2/data/info.pb
09:03:06.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:06.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:06.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:03:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:06.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.198.252:35341
09:03:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.198.252
09:03:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.198.212:46237
09:03:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.198.254:43159,127.0.198.253:44451,127.0.198.252:35341
09:03:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:06.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39947/jwks.json with env {}
09:03:06.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:06.067454 950 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.198.254" port: 43159 } has no permanent_uuid. Determining permanent_uuid...
09:03:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:06.088483 950 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } has no permanent_uuid. Determining permanent_uuid...
09:03:06.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:06.102633 950 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } has no permanent_uuid. Determining permanent_uuid...
09:03:06.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:06.105247 901 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.198.252:35341: connect: Connection refused (error 111)
09:03:06.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:06.109946 950 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.198.252:35341: Network error: Client connection negotiation failed: client connection to 127.0.198.252:35341: connect: Connection refused (error 111)
09:03:06.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:06.159543 950 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } attempt: 1
09:03:06.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:06.163863 950 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.198.252:35341: Network error: Client connection negotiation failed: client connection to 127.0.198.252:35341: connect: Connection refused (error 111)
09:03:06.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:06.232306 950 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } attempt: 2
09:03:06.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:06.236677 950 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.198.252:35341: Network error: Client connection negotiation failed: client connection to 127.0.198.252:35341: connect: Connection refused (error 111)
09:03:06.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:06.328166 950 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } attempt: 3
09:03:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:06.332329 950 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.198.252:35341: Network error: Client connection negotiation failed: client connection to 127.0.198.252:35341: connect: Connection refused (error 111)
09:03:06.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:06.376827 952 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:06.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:06.377393 952 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:06.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:06.378037 952 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:06.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:06.412809 952 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
09:03:06.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:06.413116 952 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:06.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:06.413326 952 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
09:03:06.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:06.413616 952 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
09:03:06.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:06.413821 952 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
09:03:06.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:06.413987 952 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39947/jwks.json
09:03:06.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:06.450996 952 master_runner.cc:387] Master server non-default flags:
09:03:06.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-2/logs
09:03:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.198.212:46237
09:03:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:06.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-2/data
09:03:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-2/wal
09:03:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.198.254:43159,127.0.198.253:44451,127.0.198.252:35341
09:03:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:06.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.198.252:35341
09:03:06.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:06.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:06.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:06.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39947/jwks.json
09:03:06.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:06.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:06.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-2/data/info.pb
09:03:06.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.198.252
09:03:06.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:06.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:06.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:06.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:06.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:06.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:06.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:03:06.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:03:06.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:06.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:06.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:06.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Nov 2026 08:43:21 UTC on e7f111948823
09:03:06.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14810
09:03:06.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:06.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:06.453011 952 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:06.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:06.455565 952 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:06.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:06.466614 962 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:06.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:06.475744 950 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } attempt: 4
09:03:06.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:06.481462 950 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.198.252:35341: Network error: Client connection negotiation failed: client connection to 127.0.198.252:35341: connect: Connection refused (error 111)
09:03:06.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:06.610810 875 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } attempt: 7
09:03:06.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:06.643220 875 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } has no permanent_uuid. Determining permanent_uuid...
09:03:06.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:06.652419 875 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.198.252:35341: Network error: Client connection negotiation failed: client connection to 127.0.198.252:35341: connect: Connection refused (error 111)
09:03:06.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:06.710997 875 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } attempt: 1
09:03:06.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:06.717770 875 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.198.252:35341: Network error: Client connection negotiation failed: client connection to 127.0.198.252:35341: connect: Connection refused (error 111)
09:03:06.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:06.780922 950 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } attempt: 5
09:03:06.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:06.787855 950 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.198.252:35341: Network error: Client connection negotiation failed: client connection to 127.0.198.252:35341: connect: Connection refused (error 111)
09:03:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:06.799345 875 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } attempt: 2
09:03:06.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:06.805966 875 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.198.252:35341: Network error: Client connection negotiation failed: client connection to 127.0.198.252:35341: connect: Connection refused (error 111)
09:03:06.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:06.891479 875 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } attempt: 3
09:03:06.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:06.899075 875 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.198.252:35341: Network error: Client connection negotiation failed: client connection to 127.0.198.252:35341: connect: Connection refused (error 111)
09:03:07.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.039613 875 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } attempt: 4
09:03:07.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:07.048200 875 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.198.252:35341: Network error: Client connection negotiation failed: client connection to 127.0.198.252:35341: connect: Connection refused (error 111)
09:03:07.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.331751 875 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } attempt: 5
09:03:07.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.335512 950 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } attempt: 6
09:03:07.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:07.339522 875 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.198.252:35341: Network error: Client connection negotiation failed: client connection to 127.0.198.252:35341: connect: Connection refused (error 111)
09:03:07.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:07.349273 950 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.0.198.252:35341: Network error: Client connection negotiation failed: client connection to 127.0.198.252:35341: connect: Connection refused (error 111)
09:03:07.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:06.468598 963 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:07.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:07.685052 964 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1215 milliseconds
09:03:07.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:07.685808 965 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:07.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.685801 952 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:03:07.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.689750 952 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:07.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.692330 952 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:07.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.693733 952 hybrid_clock.cc:648] HybridClock initialized: now 1794819787693686 us; error 43 us; skew 500 ppm
09:03:07.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.694672 952 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:07.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.701265 952 webserver.cc:577] Webserver started at http://127.0.198.252:35187/ using document root <none> and password file <none>
09:03:07.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.702291 952 fs_manager.cc:362] Metadata directory not provided
09:03:07.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.702529 952 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:07.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.703044 952 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:03:07.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.707371 952 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-2/data/instance:
09:03:07.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "61335ddc71144786a9932aabdae03f9f"
09:03:07.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:07 on dist-test-slave-sqmp"
09:03:07.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.708633 952 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-2/wal/instance:
09:03:07.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "61335ddc71144786a9932aabdae03f9f"
09:03:07.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:07 on dist-test-slave-sqmp"
09:03:07.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.715981 952 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
09:03:07.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.721475 977 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:03:07.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.722671 952 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
09:03:07.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.722940 952 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-2/data,/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-2/wal
09:03:07.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "61335ddc71144786a9932aabdae03f9f"
09:03:07.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:07 on dist-test-slave-sqmp"
09:03:07.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.723345 952 fs_report.cc:389] FS layout report
09:03:07.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:07.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-2/wal
09:03:07.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-2/wal
09:03:07.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-2/data/data
09:03:07.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:07.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:07.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:07.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:07.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:07.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:07.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:07.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:07.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:07.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:07.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:07.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.797111 952 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:03:07.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.798619 952 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:07.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.799007 952 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:07.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.869536 952 rpc_server.cc:307] RPC server started. Bound to: 127.0.198.252:35341
09:03:07.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.869607 1028 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.198.252:35341 every 8 connection(s)
09:03:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.872267 952 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-2/data/info.pb
09:03:07.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.878223 1029 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.881338 795 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu as pid 952
09:03:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.881788 795 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/master-2/wal/instance
09:03:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.892010 875 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } attempt: 6
09:03:07.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.890596 1029 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.198.254" port: 43159 } has no permanent_uuid. Determining permanent_uuid...
09:03:07.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.930243 1029 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } has no permanent_uuid. Determining permanent_uuid...
09:03:07.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.939900 875 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c: Bootstrap starting.
09:03:07.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.941936 1029 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } has no permanent_uuid. Determining permanent_uuid...
09:03:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.947865 875 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c: Neither blocks nor log segments found. Creating new log.
09:03:07.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.950397 875 log.cc:826] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c: Log is configured to *not* fsync() on all Append() calls
09:03:07.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.955911 875 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c: No bootstrap required, opened a new log
09:03:07.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.960933 1029 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 61335ddc71144786a9932aabdae03f9f: Bootstrap starting.
09:03:07.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.969470 1029 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 61335ddc71144786a9932aabdae03f9f: Neither blocks nor log segments found. Creating new log.
09:03:07.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.971781 1029 log.cc:826] T 00000000000000000000000000000000 P 61335ddc71144786a9932aabdae03f9f: Log is configured to *not* fsync() on all Append() calls
09:03:07.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.976866 1029 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 61335ddc71144786a9932aabdae03f9f: No bootstrap required, opened a new log
09:03:07.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.983824 875 raft_consensus.cc:360] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "defc3d7aa00d416b9b764c0b6665053c" member_type: VOTER last_known_addr { host: "127.0.198.254" port: 43159 } } peers { permanent_uuid: "ae11cc96c8a140dda8a76dbcf182c925" member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } } peers { permanent_uuid: "61335ddc71144786a9932aabdae03f9f" member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } }
09:03:07.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.984645 875 raft_consensus.cc:386] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.984921 875 raft_consensus.cc:748] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: defc3d7aa00d416b9b764c0b6665053c, State: Initialized, Role: FOLLOWER
09:03:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.985872 875 consensus_queue.cc:261] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "defc3d7aa00d416b9b764c0b6665053c" member_type: VOTER last_known_addr { host: "127.0.198.254" port: 43159 } } peers { permanent_uuid: "ae11cc96c8a140dda8a76dbcf182c925" member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } } peers { permanent_uuid: "61335ddc71144786a9932aabdae03f9f" member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } }
09:03:07.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.993614 1040 sys_catalog.cc:455] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "defc3d7aa00d416b9b764c0b6665053c" member_type: VOTER last_known_addr { host: "127.0.198.254" port: 43159 } } peers { permanent_uuid: "ae11cc96c8a140dda8a76dbcf182c925" member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } } peers { permanent_uuid: "61335ddc71144786a9932aabdae03f9f" member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } } }
09:03:07.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.994647 1040 sys_catalog.cc:458] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [sys.catalog]: This master's current role is: FOLLOWER
09:03:07.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:07.995563 875 sys_catalog.cc:565] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [sys.catalog]: configured and running, proceeding with master startup.
09:03:08.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.007248 1029 raft_consensus.cc:360] T 00000000000000000000000000000000 P 61335ddc71144786a9932aabdae03f9f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "defc3d7aa00d416b9b764c0b6665053c" member_type: VOTER last_known_addr { host: "127.0.198.254" port: 43159 } } peers { permanent_uuid: "ae11cc96c8a140dda8a76dbcf182c925" member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } } peers { permanent_uuid: "61335ddc71144786a9932aabdae03f9f" member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } }
09:03:08.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.008104 1029 raft_consensus.cc:386] T 00000000000000000000000000000000 P 61335ddc71144786a9932aabdae03f9f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:08.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.008385 1029 raft_consensus.cc:748] T 00000000000000000000000000000000 P 61335ddc71144786a9932aabdae03f9f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 61335ddc71144786a9932aabdae03f9f, State: Initialized, Role: FOLLOWER
09:03:08.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.009963 1029 consensus_queue.cc:261] T 00000000000000000000000000000000 P 61335ddc71144786a9932aabdae03f9f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "defc3d7aa00d416b9b764c0b6665053c" member_type: VOTER last_known_addr { host: "127.0.198.254" port: 43159 } } peers { permanent_uuid: "ae11cc96c8a140dda8a76dbcf182c925" member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } } peers { permanent_uuid: "61335ddc71144786a9932aabdae03f9f" member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } }
09:03:08.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.013311 1048 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61335ddc71144786a9932aabdae03f9f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "defc3d7aa00d416b9b764c0b6665053c" member_type: VOTER last_known_addr { host: "127.0.198.254" port: 43159 } } peers { permanent_uuid: "ae11cc96c8a140dda8a76dbcf182c925" member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } } peers { permanent_uuid: "61335ddc71144786a9932aabdae03f9f" member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } } }
09:03:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.015064 1048 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61335ddc71144786a9932aabdae03f9f [sys.catalog]: This master's current role is: FOLLOWER
09:03:08.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.015952 1029 sys_catalog.cc:565] T 00000000000000000000000000000000 P 61335ddc71144786a9932aabdae03f9f [sys.catalog]: configured and running, proceeding with master startup.
09:03:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:08.030009 1052 catalog_manager.cc:1609] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:03:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:08.030645 1052 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:03:08.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:08.055243 983 tablet.cc:2441] T 00000000000000000000000000000000 P 61335ddc71144786a9932aabdae03f9f: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:03:08.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.064468 1065 catalog_manager.cc:820] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
09:03:08.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.028008 807 master_runner.cc:187] Error getting master registration for 127.0.198.253:44451: OK, instance_id { permanent_uuid: "ae11cc96c8a140dda8a76dbcf182c925" instance_seqno: 1794819786024798 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:03:08.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.066762 807 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
09:03:08.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.074865 952 master_runner.cc:187] Error getting master registration for 127.0.198.253:44451: OK, instance_id { permanent_uuid: "ae11cc96c8a140dda8a76dbcf182c925" instance_seqno: 1794819786024798 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:03:08.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.095103 952 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
09:03:08.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:08.173503 829 tablet.cc:2441] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:03:08.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.305732 1040 raft_consensus.cc:499] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:03:08.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.306133 1040 raft_consensus.cc:521] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "defc3d7aa00d416b9b764c0b6665053c" member_type: VOTER last_known_addr { host: "127.0.198.254" port: 43159 } } peers { permanent_uuid: "ae11cc96c8a140dda8a76dbcf182c925" member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } } peers { permanent_uuid: "61335ddc71144786a9932aabdae03f9f" member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } }
09:03:08.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.308290 1040 leader_election.cc:290] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ae11cc96c8a140dda8a76dbcf182c925 (127.0.198.253:44451), 61335ddc71144786a9932aabdae03f9f (127.0.198.252:35341)
09:03:08.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.309867 1004 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "defc3d7aa00d416b9b764c0b6665053c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "61335ddc71144786a9932aabdae03f9f" is_pre_election: true
09:03:08.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.310742 1004 raft_consensus.cc:2490] T 00000000000000000000000000000000 P 61335ddc71144786a9932aabdae03f9f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate defc3d7aa00d416b9b764c0b6665053c in term 0.
09:03:08.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.312086 826 leader_election.cc:304] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 61335ddc71144786a9932aabdae03f9f, defc3d7aa00d416b9b764c0b6665053c; no voters:
09:03:08.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.313040 1040 raft_consensus.cc:2826] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [term 0 FOLLOWER]: Leader pre-election won for term 1
09:03:08.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.313426 1040 raft_consensus.cc:499] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:03:08.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.313797 1040 raft_consensus.cc:3096] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [term 0 FOLLOWER]: Advancing to term 1
09:03:08.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.321703 1040 raft_consensus.cc:521] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "defc3d7aa00d416b9b764c0b6665053c" member_type: VOTER last_known_addr { host: "127.0.198.254" port: 43159 } } peers { permanent_uuid: "ae11cc96c8a140dda8a76dbcf182c925" member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } } peers { permanent_uuid: "61335ddc71144786a9932aabdae03f9f" member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } }
09:03:08.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.323729 1040 leader_election.cc:290] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [CANDIDATE]: Term 1 election: Requested vote from peers ae11cc96c8a140dda8a76dbcf182c925 (127.0.198.253:44451), 61335ddc71144786a9932aabdae03f9f (127.0.198.252:35341)
09:03:08.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.324805 1004 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "defc3d7aa00d416b9b764c0b6665053c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "61335ddc71144786a9932aabdae03f9f"
09:03:08.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.325289 1004 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 61335ddc71144786a9932aabdae03f9f [term 0 FOLLOWER]: Advancing to term 1
09:03:08.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.331885 1004 raft_consensus.cc:2490] T 00000000000000000000000000000000 P 61335ddc71144786a9932aabdae03f9f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate defc3d7aa00d416b9b764c0b6665053c in term 1.
09:03:08.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.333865 826 leader_election.cc:304] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 61335ddc71144786a9932aabdae03f9f, defc3d7aa00d416b9b764c0b6665053c; no voters:
09:03:08.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.335078 1040 raft_consensus.cc:2826] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [term 1 FOLLOWER]: Leader election won for term 1
09:03:08.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.337280 1040 raft_consensus.cc:705] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [term 1 LEADER]: Becoming Leader. State: Replica: defc3d7aa00d416b9b764c0b6665053c, State: Running, Role: LEADER
09:03:08.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.338217 1040 consensus_queue.cc:238] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "defc3d7aa00d416b9b764c0b6665053c" member_type: VOTER last_known_addr { host: "127.0.198.254" port: 43159 } } peers { permanent_uuid: "ae11cc96c8a140dda8a76dbcf182c925" member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } } peers { permanent_uuid: "61335ddc71144786a9932aabdae03f9f" member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } }
09:03:08.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.352419 1068 consensus_queue.cc:1049] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [LEADER]: Connected to new peer: Peer: permanent_uuid: "61335ddc71144786a9932aabdae03f9f" member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
09:03:08.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.352494 1066 sys_catalog.cc:455] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [sys.catalog]: SysCatalogTable state changed. Reason: New leader defc3d7aa00d416b9b764c0b6665053c. Latest consensus state: current_term: 1 leader_uuid: "defc3d7aa00d416b9b764c0b6665053c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "defc3d7aa00d416b9b764c0b6665053c" member_type: VOTER last_known_addr { host: "127.0.198.254" port: 43159 } } peers { permanent_uuid: "ae11cc96c8a140dda8a76dbcf182c925" member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } } peers { permanent_uuid: "61335ddc71144786a9932aabdae03f9f" member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } } }
09:03:08.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.353318 1066 sys_catalog.cc:458] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [sys.catalog]: This master's current role is: LEADER
09:03:08.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.359962 1070 catalog_manager.cc:1526] Loading table and tablet metadata into memory...
09:03:08.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.367688 1070 catalog_manager.cc:1535] Initializing Kudu cluster ID...
09:03:08.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.371515 1048 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61335ddc71144786a9932aabdae03f9f [sys.catalog]: SysCatalogTable state changed. Reason: New leader defc3d7aa00d416b9b764c0b6665053c. Latest consensus state: current_term: 1 leader_uuid: "defc3d7aa00d416b9b764c0b6665053c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "defc3d7aa00d416b9b764c0b6665053c" member_type: VOTER last_known_addr { host: "127.0.198.254" port: 43159 } } peers { permanent_uuid: "ae11cc96c8a140dda8a76dbcf182c925" member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } } peers { permanent_uuid: "61335ddc71144786a9932aabdae03f9f" member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } } }
09:03:08.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.372082 1048 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61335ddc71144786a9932aabdae03f9f [sys.catalog]: This master's current role is: FOLLOWER
09:03:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.380098 1040 sys_catalog.cc:455] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "defc3d7aa00d416b9b764c0b6665053c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "defc3d7aa00d416b9b764c0b6665053c" member_type: VOTER last_known_addr { host: "127.0.198.254" port: 43159 } } peers { permanent_uuid: "ae11cc96c8a140dda8a76dbcf182c925" member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } } peers { permanent_uuid: "61335ddc71144786a9932aabdae03f9f" member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } } }
09:03:08.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.380736 1040 sys_catalog.cc:458] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [sys.catalog]: This master's current role is: LEADER
09:03:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.387638 1048 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61335ddc71144786a9932aabdae03f9f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "defc3d7aa00d416b9b764c0b6665053c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "defc3d7aa00d416b9b764c0b6665053c" member_type: VOTER last_known_addr { host: "127.0.198.254" port: 43159 } } peers { permanent_uuid: "ae11cc96c8a140dda8a76dbcf182c925" member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } } peers { permanent_uuid: "61335ddc71144786a9932aabdae03f9f" member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } } }
09:03:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.388270 1048 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61335ddc71144786a9932aabdae03f9f [sys.catalog]: This master's current role is: FOLLOWER
09:03:08.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.408344 1070 catalog_manager.cc:1398] Generated new cluster ID: 56bd3c3a351c4984a64a7ff1bfd26e23
09:03:08.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.409842 950 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } attempt: 7
09:03:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.409657 1070 catalog_manager.cc:1546] Initializing Kudu internal certificate authority...
09:03:08.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.455817 950 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ae11cc96c8a140dda8a76dbcf182c925: Bootstrap starting.
09:03:08.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.456763 1070 catalog_manager.cc:1421] Generated new certificate authority record
09:03:08.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.459259 1070 catalog_manager.cc:1555] Loading token signing keys...
09:03:08.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.468307 950 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ae11cc96c8a140dda8a76dbcf182c925: Neither blocks nor log segments found. Creating new log.
09:03:08.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.470815 950 log.cc:826] T 00000000000000000000000000000000 P ae11cc96c8a140dda8a76dbcf182c925: Log is configured to *not* fsync() on all Append() calls
09:03:08.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.482478 950 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ae11cc96c8a140dda8a76dbcf182c925: No bootstrap required, opened a new log
09:03:08.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.486260 1070 catalog_manager.cc:6221] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c: Generated new TSK 0
09:03:08.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.487661 1070 catalog_manager.cc:1565] Initializing in-progress tserver states...
09:03:08.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.509508 950 raft_consensus.cc:360] T 00000000000000000000000000000000 P ae11cc96c8a140dda8a76dbcf182c925 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "defc3d7aa00d416b9b764c0b6665053c" member_type: VOTER last_known_addr { host: "127.0.198.254" port: 43159 } } peers { permanent_uuid: "ae11cc96c8a140dda8a76dbcf182c925" member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } } peers { permanent_uuid: "61335ddc71144786a9932aabdae03f9f" member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } }
09:03:08.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.510170 950 raft_consensus.cc:386] T 00000000000000000000000000000000 P ae11cc96c8a140dda8a76dbcf182c925 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:08.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.510507 950 raft_consensus.cc:748] T 00000000000000000000000000000000 P ae11cc96c8a140dda8a76dbcf182c925 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ae11cc96c8a140dda8a76dbcf182c925, State: Initialized, Role: FOLLOWER
09:03:08.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.511415 950 consensus_queue.cc:261] T 00000000000000000000000000000000 P ae11cc96c8a140dda8a76dbcf182c925 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "defc3d7aa00d416b9b764c0b6665053c" member_type: VOTER last_known_addr { host: "127.0.198.254" port: 43159 } } peers { permanent_uuid: "ae11cc96c8a140dda8a76dbcf182c925" member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } } peers { permanent_uuid: "61335ddc71144786a9932aabdae03f9f" member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } }
09:03:08.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.514387 1078 sys_catalog.cc:455] T 00000000000000000000000000000000 P ae11cc96c8a140dda8a76dbcf182c925 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "defc3d7aa00d416b9b764c0b6665053c" member_type: VOTER last_known_addr { host: "127.0.198.254" port: 43159 } } peers { permanent_uuid: "ae11cc96c8a140dda8a76dbcf182c925" member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } } peers { permanent_uuid: "61335ddc71144786a9932aabdae03f9f" member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } } }
09:03:08.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.515223 1078 sys_catalog.cc:458] T 00000000000000000000000000000000 P ae11cc96c8a140dda8a76dbcf182c925 [sys.catalog]: This master's current role is: FOLLOWER
09:03:08.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.516372 950 sys_catalog.cc:565] T 00000000000000000000000000000000 P ae11cc96c8a140dda8a76dbcf182c925 [sys.catalog]: configured and running, proceeding with master startup.
09:03:08.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.516960 923 raft_consensus.cc:3096] T 00000000000000000000000000000000 P ae11cc96c8a140dda8a76dbcf182c925 [term 0 FOLLOWER]: Advancing to term 1
09:03:08.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.517066 925 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "defc3d7aa00d416b9b764c0b6665053c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ae11cc96c8a140dda8a76dbcf182c925" is_pre_election: true
09:03:08.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.517884 924 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "defc3d7aa00d416b9b764c0b6665053c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ae11cc96c8a140dda8a76dbcf182c925"
09:03:08.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.535029 1066 consensus_queue.cc:1049] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ae11cc96c8a140dda8a76dbcf182c925" member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:03:08.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.540820 795 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:08.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:08.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-0/wal
09:03:08.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-0/data
09:03:08.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:08.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:08.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:08.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:08.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:08.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:08.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-0/logs
09:03:08.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-0/data/info.pb
09:03:08.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:08.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:08.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:08.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:08.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:08.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:08.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:08.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:08.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:03:08.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:08.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.198.193:0
09:03:08.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.198.193
09:03:08.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.198.193
09:03:08.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:08.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.198.254:43159,127.0.198.253:44451,127.0.198.252:35341
09:03:08.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.198.212:46237
09:03:08.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:08.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:08.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
09:03:08.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.623893 1091 sys_catalog.cc:455] T 00000000000000000000000000000000 P ae11cc96c8a140dda8a76dbcf182c925 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "defc3d7aa00d416b9b764c0b6665053c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "defc3d7aa00d416b9b764c0b6665053c" member_type: VOTER last_known_addr { host: "127.0.198.254" port: 43159 } } peers { permanent_uuid: "ae11cc96c8a140dda8a76dbcf182c925" member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } } peers { permanent_uuid: "61335ddc71144786a9932aabdae03f9f" member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } } }
09:03:08.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.624980 1091 sys_catalog.cc:458] T 00000000000000000000000000000000 P ae11cc96c8a140dda8a76dbcf182c925 [sys.catalog]: This master's current role is: FOLLOWER
09:03:08.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.626299 1078 sys_catalog.cc:455] T 00000000000000000000000000000000 P ae11cc96c8a140dda8a76dbcf182c925 [sys.catalog]: SysCatalogTable state changed. Reason: New leader defc3d7aa00d416b9b764c0b6665053c. Latest consensus state: current_term: 1 leader_uuid: "defc3d7aa00d416b9b764c0b6665053c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "defc3d7aa00d416b9b764c0b6665053c" member_type: VOTER last_known_addr { host: "127.0.198.254" port: 43159 } } peers { permanent_uuid: "ae11cc96c8a140dda8a76dbcf182c925" member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } } peers { permanent_uuid: "61335ddc71144786a9932aabdae03f9f" member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } } }
09:03:08.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.627780 1078 sys_catalog.cc:458] T 00000000000000000000000000000000 P ae11cc96c8a140dda8a76dbcf182c925 [sys.catalog]: This master's current role is: FOLLOWER
09:03:08.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.640422 1095 catalog_manager.cc:1310] Loaded cluster ID: 56bd3c3a351c4984a64a7ff1bfd26e23
09:03:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.641090 1095 catalog_manager.cc:1603] T 00000000000000000000000000000000 P ae11cc96c8a140dda8a76dbcf182c925: loading cluster ID for follower catalog manager: success
09:03:08.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.648916 1095 catalog_manager.cc:1625] T 00000000000000000000000000000000 P ae11cc96c8a140dda8a76dbcf182c925: acquiring CA information for follower catalog manager: success
09:03:08.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.654690 1095 catalog_manager.cc:1653] T 00000000000000000000000000000000 P ae11cc96c8a140dda8a76dbcf182c925: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:03:08.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.927300 1066 sys_catalog.cc:455] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "defc3d7aa00d416b9b764c0b6665053c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "defc3d7aa00d416b9b764c0b6665053c" member_type: VOTER last_known_addr { host: "127.0.198.254" port: 43159 } } peers { permanent_uuid: "ae11cc96c8a140dda8a76dbcf182c925" member_type: VOTER last_known_addr { host: "127.0.198.253" port: 44451 } } peers { permanent_uuid: "61335ddc71144786a9932aabdae03f9f" member_type: VOTER last_known_addr { host: "127.0.198.252" port: 35341 } } }
09:03:08.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:08.927999 1066 sys_catalog.cc:458] T 00000000000000000000000000000000 P defc3d7aa00d416b9b764c0b6665053c [sys.catalog]: This master's current role is: LEADER
09:03:09.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:09.060777 1086 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:09.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:09.061904 1086 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:09.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:09.062909 1086 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:09.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:09.085788 1065 catalog_manager.cc:1310] Loaded cluster ID: 56bd3c3a351c4984a64a7ff1bfd26e23
09:03:09.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:09.086426 1065 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 61335ddc71144786a9932aabdae03f9f: loading cluster ID for follower catalog manager: success
09:03:09.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:09.102015 1065 catalog_manager.cc:1625] T 00000000000000000000000000000000 P 61335ddc71144786a9932aabdae03f9f: acquiring CA information for follower catalog manager: success
09:03:09.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:09.107815 1065 catalog_manager.cc:1653] T 00000000000000000000000000000000 P 61335ddc71144786a9932aabdae03f9f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:03:09.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:09.124869 1086 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:09.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:09.126503 1086 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.198.193
09:03:09.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:09.192031 1086 tablet_server_runner.cc:78] Tablet server non-default flags:
09:03:09.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-0/logs
09:03:09.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:09.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:09.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:09.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.198.212:46237
09:03:09.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:09.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:09.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-0/data
09:03:09.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-0/wal
09:03:09.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:09.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:09.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.198.193:0
09:03:09.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:09.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:09.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:09.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-0/data/info.pb
09:03:09.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.198.193
09:03:09.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:09.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.198.254:43159,127.0.198.253:44451,127.0.198.252:35341
09:03:09.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:09.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:09.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:09.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:09.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:09.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.198.193
09:03:09.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:03:09.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:03:09.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:09.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:09.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:09.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Nov 2026 08:43:21 UTC on e7f111948823
09:03:09.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14810
09:03:09.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:09.197069 1086 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:09.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:09.199416 1086 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:09.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:09.234400 1104 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:11.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:10.621356 1103 debug-util.cc:398] Leaking SignalData structure 0x7b0800029220 after lost signal to thread 1086
09:03:11.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:09.235100 1105 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:11.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:11.664456 1103 kernel_stack_watchdog.cc:198] Thread 1086 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 387ms:
09:03:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
09:03:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
09:03:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
09:03:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
09:03:11.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:11.666046 1086 thread.cc:642] GCE (cloud detector) Time spent creating pthread: real 2.433s user 0.485s sys 0.855s
09:03:11.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:11.666399 1086 thread.cc:609] GCE (cloud detector) Time spent starting thread: real 2.434s user 0.485s sys 0.855s
09:03:11.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.673431 1086 server_base.cc:1061] running on GCE node
09:03:11.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:11.673600 1109 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:11.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.674857 1086 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:11.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.677042 1086 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:11.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.678404 1086 hybrid_clock.cc:648] HybridClock initialized: now 1794819791678358 us; error 44 us; skew 500 ppm
09:03:11.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.679234 1086 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:11.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.685415 1086 webserver.cc:577] Webserver started at http://127.0.198.193:35543/ using document root <none> and password file <none>
09:03:11.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.686489 1086 fs_manager.cc:362] Metadata directory not provided
09:03:11.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.686975 1086 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:11.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.687515 1086 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:03:11.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.691877 1086 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-0/data/instance:
09:03:11.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "731454a42ebf4fb989631b5a6d3c3afe"
09:03:11.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:11 on dist-test-slave-sqmp"
09:03:11.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.693219 1086 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-0/wal/instance:
09:03:11.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "731454a42ebf4fb989631b5a6d3c3afe"
09:03:11.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:11 on dist-test-slave-sqmp"
09:03:11.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.701535 1086 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.000s
09:03:11.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.707259 1114 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:03:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.708503 1086 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
09:03:11.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.708837 1086 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-0/data,/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-0/wal
09:03:11.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "731454a42ebf4fb989631b5a6d3c3afe"
09:03:11.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:11 on dist-test-slave-sqmp"
09:03:11.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.709260 1086 fs_report.cc:389] FS layout report
09:03:11.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:11.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-0/wal
09:03:11.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-0/wal
09:03:11.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-0/data/data
09:03:11.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:11.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:11.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:11.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:11.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:11.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:11.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:11.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:11.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:11.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:11.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:11.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.762856 1086 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:03:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.764281 1086 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.764693 1086 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.767933 1086 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
09:03:11.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.772244 1086 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:03:11.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.772437 1086 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:03:11.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.772646 1086 ts_tablet_manager.cc:662] Registered 0 tablets
09:03:11.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.772776 1086 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:03:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.949896 1086 rpc_server.cc:307] RPC server started. Bound to: 127.0.198.193:42225
09:03:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.950053 1227 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.198.193:42225 every 8 connection(s)
09:03:11.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.952646 1086 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-0/data/info.pb
09:03:11.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.955624 795 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu as pid 1086
09:03:11.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.956211 795 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-0/wal/instance
09:03:11.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:11.969136 795 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:11.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:11.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-1/wal
09:03:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-1/data
09:03:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-1/logs
09:03:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-1/data/info.pb
09:03:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:11.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:03:11.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:11.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.198.194:0
09:03:11.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.198.194
09:03:11.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.198.194
09:03:11.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:11.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.198.254:43159,127.0.198.253:44451,127.0.198.252:35341
09:03:11.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.198.212:46237
09:03:11.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:11.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:11.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
09:03:12.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:12.036585 1230 heartbeater.cc:345] Connected to a master server at 127.0.198.254:43159
09:03:12.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:12.037096 1230 heartbeater.cc:462] Registering TS with master...
09:03:12.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:12.038381 1230 heartbeater.cc:508] Master 127.0.198.254:43159 requested a full tablet report, sending...
09:03:12.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:12.042356 840 ts_manager.cc:210] Registered new tserver with Master: 731454a42ebf4fb989631b5a6d3c3afe (127.0.198.193:42225)
09:03:12.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:12.045071 1228 heartbeater.cc:345] Connected to a master server at 127.0.198.252:35341
09:03:12.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:12.044951 840 master_service.cc:507] Signed X509 certificate for tserver {username='slave'} at 127.0.198.193:58479
09:03:12.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:12.045431 1228 heartbeater.cc:462] Registering TS with master...
09:03:12.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:12.046294 1228 heartbeater.cc:508] Master 127.0.198.252:35341 requested a full tablet report, sending...
09:03:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:12.049989 994 ts_manager.cc:210] Registered new tserver with Master: 731454a42ebf4fb989631b5a6d3c3afe (127.0.198.193:42225)
09:03:12.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:12.052835 1229 heartbeater.cc:345] Connected to a master server at 127.0.198.253:44451
09:03:12.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:12.053141 1229 heartbeater.cc:462] Registering TS with master...
09:03:12.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:12.054037 1229 heartbeater.cc:508] Master 127.0.198.253:44451 requested a full tablet report, sending...
09:03:12.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:12.057416 915 ts_manager.cc:210] Registered new tserver with Master: 731454a42ebf4fb989631b5a6d3c3afe (127.0.198.193:42225)
09:03:12.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:12.335803 1233 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:12.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:12.336247 1233 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:12.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:12.336685 1233 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:12.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:12.369899 1233 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:12.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:12.370808 1233 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.198.194
09:03:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:12.405681 1233 tablet_server_runner.cc:78] Tablet server non-default flags:
09:03:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-1/logs
09:03:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.198.212:46237
09:03:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-1/data
09:03:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-1/wal
09:03:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.198.194:0
09:03:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-1/data/info.pb
09:03:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.198.194
09:03:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.198.254:43159,127.0.198.253:44451,127.0.198.252:35341
09:03:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:12.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:12.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:12.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:12.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:12.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.198.194
09:03:12.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:03:12.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:03:12.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:12.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:12.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:12.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Nov 2026 08:43:21 UTC on e7f111948823
09:03:12.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14810
09:03:12.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:12.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:12.407042 1233 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:12.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:12.409209 1233 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:12.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:12.423002 1245 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:13.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:13.056064 1230 heartbeater.cc:500] Master 127.0.198.254:43159 was elected leader, sending a full tablet report...
09:03:14.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:13.841207 1244 debug-util.cc:398] Leaking SignalData structure 0x7b0800029220 after lost signal to thread 1233
09:03:14.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:14.013134 1233 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.590s user 0.507s sys 1.062s
09:03:14.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:14.014338 1233 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.591s user 0.507s sys 1.062s
09:03:14.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:12.424163 1246 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:14.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.015507 1233 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:03:14.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:14.015615 1247 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1590 milliseconds
09:03:14.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:14.015627 1248 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:14.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.020460 1233 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:14.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.022473 1233 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:14.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.023809 1233 hybrid_clock.cc:648] HybridClock initialized: now 1794819794023777 us; error 49 us; skew 500 ppm
09:03:14.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.024577 1233 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:14.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.031874 1233 webserver.cc:577] Webserver started at http://127.0.198.194:38669/ using document root <none> and password file <none>
09:03:14.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.033056 1233 fs_manager.cc:362] Metadata directory not provided
09:03:14.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.033335 1233 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:14.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.033900 1233 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:03:14.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.039891 1233 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-1/data/instance:
09:03:14.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a6436435636d4092b71d319489bd0419"
09:03:14.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:14 on dist-test-slave-sqmp"
09:03:14.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.041337 1233 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-1/wal/instance:
09:03:14.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a6436435636d4092b71d319489bd0419"
09:03:14.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:14 on dist-test-slave-sqmp"
09:03:14.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.048908 1233 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
09:03:14.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.055305 1256 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:03:14.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.056504 1233 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
09:03:14.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.056843 1233 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-1/data,/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-1/wal
09:03:14.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a6436435636d4092b71d319489bd0419"
09:03:14.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:14 on dist-test-slave-sqmp"
09:03:14.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.057210 1233 fs_report.cc:389] FS layout report
09:03:14.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:14.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-1/wal
09:03:14.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-1/wal
09:03:14.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-1/data/data
09:03:14.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:14.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:14.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:14.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:14.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:14.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:14.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:14.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:14.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:14.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:14.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:14.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.111517 1233 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:03:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.112989 1233 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:14.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.113523 1233 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:14.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.116074 1233 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
09:03:14.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.120359 1233 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:03:14.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.120594 1233 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:03:14.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.120930 1233 ts_tablet_manager.cc:662] Registered 0 tablets
09:03:14.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.121145 1233 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:03:14.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:14.133713 1025 debug-util.cc:398] Leaking SignalData structure 0x7b08000ab740 after lost signal to thread 957
09:03:14.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:14.134390 1025 debug-util.cc:398] Leaking SignalData structure 0x7b0800074e20 after lost signal to thread 1028
09:03:14.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.306945 1233 rpc_server.cc:307] RPC server started. Bound to: 127.0.198.194:42799
09:03:14.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.307035 1369 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.198.194:42799 every 8 connection(s)
09:03:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.309734 1233 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-1/data/info.pb
09:03:14.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.313014 795 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu as pid 1233
09:03:14.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.313840 795 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-1/wal/instance
09:03:14.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.330569 795 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:14.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:14.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-2/wal
09:03:14.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-2/data
09:03:14.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:14.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:14.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:14.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:14.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:14.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:14.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-2/logs
09:03:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-2/data/info.pb
09:03:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:03:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.198.195:0
09:03:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.198.195
09:03:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.198.195
09:03:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.198.254:43159,127.0.198.253:44451,127.0.198.252:35341
09:03:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.198.212:46237
09:03:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
09:03:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.398998 1374 heartbeater.cc:345] Connected to a master server at 127.0.198.254:43159
09:03:14.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.399940 1374 heartbeater.cc:462] Registering TS with master...
09:03:14.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.401538 1374 heartbeater.cc:508] Master 127.0.198.254:43159 requested a full tablet report, sending...
09:03:14.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.405329 840 ts_manager.cc:210] Registered new tserver with Master: a6436435636d4092b71d319489bd0419 (127.0.198.194:42799)
09:03:14.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.405586 1370 heartbeater.cc:345] Connected to a master server at 127.0.198.252:35341
09:03:14.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.405900 1370 heartbeater.cc:462] Registering TS with master...
09:03:14.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.406778 1370 heartbeater.cc:508] Master 127.0.198.252:35341 requested a full tablet report, sending...
09:03:14.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.407248 840 master_service.cc:507] Signed X509 certificate for tserver {username='slave'} at 127.0.198.194:43217
09:03:14.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.409530 994 ts_manager.cc:210] Registered new tserver with Master: a6436435636d4092b71d319489bd0419 (127.0.198.194:42799)
09:03:14.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.414364 1371 heartbeater.cc:345] Connected to a master server at 127.0.198.253:44451
09:03:14.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.414696 1371 heartbeater.cc:462] Registering TS with master...
09:03:14.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.415303 1371 heartbeater.cc:508] Master 127.0.198.253:44451 requested a full tablet report, sending...
09:03:14.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.418174 915 ts_manager.cc:210] Registered new tserver with Master: a6436435636d4092b71d319489bd0419 (127.0.198.194:42799)
09:03:14.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:14.709897 1377 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:14.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:14.710439 1377 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:14.711046 1377 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:14.743450 1377 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:14.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:14.744611 1377 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.198.195
09:03:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.784199 1377 tablet_server_runner.cc:78] Tablet server non-default flags:
09:03:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-2/logs
09:03:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.198.212:46237
09:03:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-2/data
09:03:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-2/wal
09:03:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.198.195:0
09:03:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-2/data/info.pb
09:03:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.198.195
09:03:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:14.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.198.254:43159,127.0.198.253:44451,127.0.198.252:35341
09:03:14.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:14.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:14.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:14.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:14.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:14.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.198.195
09:03:14.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:03:14.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:03:14.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:14.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:14.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:14.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Nov 2026 08:43:21 UTC on e7f111948823
09:03:14.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14810
09:03:14.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:14.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.785995 1377 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:14.788065 1377 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:14.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:14.801522 1387 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:15.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:15.363440 946 debug-util.cc:398] Leaking SignalData structure 0x7b080009d200 after lost signal to thread 881
09:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:15.411661 1374 heartbeater.cc:500] Master 127.0.198.254:43159 was elected leader, sending a full tablet report...
09:03:16.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:14.805477 1390 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:16.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:14.803918 1388 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:16.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:16.449041 1389 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1640 milliseconds
09:03:16.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.449180 1377 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:03:16.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.450809 1377 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.454643 1377 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:16.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.456280 1377 hybrid_clock.cc:648] HybridClock initialized: now 1794819796456210 us; error 81 us; skew 500 ppm
09:03:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.457469 1377 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:16.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.469496 1377 webserver.cc:577] Webserver started at http://127.0.198.195:45937/ using document root <none> and password file <none>
09:03:16.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.470773 1377 fs_manager.cc:362] Metadata directory not provided
09:03:16.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.471060 1377 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:16.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.471634 1377 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:03:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.478564 1377 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-2/data/instance:
09:03:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ffe3a388a609438ab37946413b5632d8"
09:03:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:16 on dist-test-slave-sqmp"
09:03:16.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.480152 1377 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-2/wal/instance:
09:03:16.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ffe3a388a609438ab37946413b5632d8"
09:03:16.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:16 on dist-test-slave-sqmp"
09:03:16.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.490213 1377 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.011s sys 0.000s
09:03:16.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.498322 1398 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:03:16.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.499742 1377 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
09:03:16.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.500133 1377 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-2/data,/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-2/wal
09:03:16.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ffe3a388a609438ab37946413b5632d8"
09:03:16.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:16 on dist-test-slave-sqmp"
09:03:16.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.500588 1377 fs_report.cc:389] FS layout report
09:03:16.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:16.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-2/wal
09:03:16.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-2/wal
09:03:16.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-2/data/data
09:03:16.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:16.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:16.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:16.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:16.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:16.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:16.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:16.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:16.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:16.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.566370 1377 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:03:16.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.568423 1377 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:16.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.569023 1377 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:16.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.572862 1377 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
09:03:16.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.579147 1377 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:03:16.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.579424 1377 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:03:16.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.579715 1377 ts_tablet_manager.cc:662] Registered 0 tablets
09:03:16.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.579963 1377 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:03:16.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.768832 1377 rpc_server.cc:307] RPC server started. Bound to: 127.0.198.195:39837
09:03:16.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.768985 1511 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.198.195:39837 every 8 connection(s)
09:03:16.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.772877 1377 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-2/data/info.pb
09:03:16.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.781311 795 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu as pid 1377
09:03:16.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.782761 795 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster14559464929593166521/ts-2/wal/instance
09:03:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.900593 1516 heartbeater.cc:345] Connected to a master server at 127.0.198.254:43159
09:03:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.901122 1516 heartbeater.cc:462] Registering TS with master...
09:03:16.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.902493 1516 heartbeater.cc:508] Master 127.0.198.254:43159 requested a full tablet report, sending...
09:03:16.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.903931 1512 heartbeater.cc:345] Connected to a master server at 127.0.198.252:35341
09:03:16.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.904325 1512 heartbeater.cc:462] Registering TS with master...
09:03:16.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.905172 1512 heartbeater.cc:508] Master 127.0.198.252:35341 requested a full tablet report, sending...
09:03:16.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.906044 840 ts_manager.cc:210] Registered new tserver with Master: ffe3a388a609438ab37946413b5632d8 (127.0.198.195:39837)
09:03:16.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.908017 840 master_service.cc:507] Signed X509 certificate for tserver {username='slave'} at 127.0.198.195:48503
09:03:16.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.908278 994 ts_manager.cc:210] Registered new tserver with Master: ffe3a388a609438ab37946413b5632d8 (127.0.198.195:39837)
09:03:16.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.914548 1513 heartbeater.cc:345] Connected to a master server at 127.0.198.253:44451
09:03:16.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.914870 1513 heartbeater.cc:462] Registering TS with master...
09:03:16.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.916301 1513 heartbeater.cc:508] Master 127.0.198.253:44451 requested a full tablet report, sending...
09:03:16.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.919852 915 ts_manager.cc:210] Registered new tserver with Master: ffe3a388a609438ab37946413b5632d8 (127.0.198.195:39837)
09:03:16.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:16.925232 795 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
09:03:16.941 [DEBUG - main] (MiniKuduCluster.java:195) Response:
09:03:16.942 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
09:03:16.951 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.0.198.254"
port: 43159
}
bound_http_address {
host: "127.0.198.254"
port: 46077
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.0.198.253"
port: 44451
}
bound_http_address {
host: "127.0.198.253"
port: 43805
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.0.198.252"
port: 35341
}
bound_http_address {
host: "127.0.198.252"
port: 35187
}
}
}
09:03:16.955 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
09:03:16.958 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.0.198.193"
port: 42225
}
bound_http_address {
host: "127.0.198.193"
port: 35543
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.0.198.194"
port: 42799
}
bound_http_address {
host: "127.0.198.194"
port: 38669
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.0.198.195"
port: 39837
}
bound_http_address {
host: "127.0.198.195"
port: 45937
}
}
}
09:03:16.958 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
09:03:17.155 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
09:03:17.219 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
09:03:17.375 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskQWUX02/test-tmp
09:03:17.376 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
09:03:17.377 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskQWUX02/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
09:03:17.386 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
09:03:17.806 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:03:17.807 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
09:03:17.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:17.812177 1524 mini_oidc.cc:150] Starting JWKS server
09:03:17.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:17.816370 1524 webserver.cc:577] Webserver started at http://127.0.0.1:37653/ using document root <none> and password file <none>
09:03:17.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:17.817651 1524 mini_oidc.cc:186] Starting OIDC Discovery server
09:03:17.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:17.819975 1524 webserver.cc:577] Webserver started at http://127.0.0.1:42541/ using document root <none> and password file <none>
09:03:17.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
09:03:17.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/principal' for realm 'KRBTEST.COM',
09:03:17.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
09:03:17.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:17 dist-test-slave-sqmp krb5kdc[1536](info): setting up network...
09:03:17.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
09:03:17.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:17 dist-test-slave-sqmp krb5kdc[1536](info): set up 2 sockets
09:03:17.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:17 dist-test-slave-sqmp krb5kdc[1536](info): commencing operation
09:03:17.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
09:03:17.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:17.918956 1516 heartbeater.cc:500] Master 127.0.198.254:43159 was elected leader, sending a full tablet report...
09:03:19.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:19.933043 1524 mini_kdc.cc:121] Time spent starting KDC: real 2.075s user 0.002s sys 0.018s
09:03:19.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
09:03:19.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
09:03:19.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
09:03:19.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
09:03:19.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
09:03:19.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
09:03:20.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
09:03:20.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
09:03:20.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
09:03:20.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
09:03:20.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/test-user.keytab.
09:03:20.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/test-user.keytab.
09:03:20.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:20 dist-test-slave-sqmp krb5kdc[1536](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819800, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:20.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
09:03:20.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-16T09:03:20Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
09:03:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-16T09:03:20Z Disabled control of system clock
09:03:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.125.62@KRBTEST.COM; defaulting to no policy
09:03:20.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:20.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.125.62@KRBTEST.COM" created.
09:03:20.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:20.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.125.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:20.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.125.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:20.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.1.125.62@KRBTEST.COM; defaulting to no policy
09:03:20.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:20.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.1.125.62@KRBTEST.COM" created.
09:03:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.125.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.125.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:20.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:20.147203 1524 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:20.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:20.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/wal
09:03:20.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/data
09:03:20.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:20.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:20.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:20.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:20.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:20.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:20.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/logs
09:03:20.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/data/info.pb
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.125.62:46583
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.125.62
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.125.62
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.125.20:41439
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.125.62:46583,127.1.125.61:32775,127.1.125.60:44533
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37653/jwks.json
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
09:03:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:03:20.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:20.467649 1557 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:20.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:20.468359 1557 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:20.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:20.469316 1557 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:20.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:20.502964 1557 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
09:03:20.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:20.503263 1557 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
09:03:20.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:20.503690 1557 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:20.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:20.504035 1557 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
09:03:20.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:20.504383 1557 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
09:03:20.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:20.504720 1557 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
09:03:20.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:20.504966 1557 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37653/jwks.json
09:03:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:20.544575 1557 master_runner.cc:387] Master server non-default flags:
09:03:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/logs
09:03:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.125.20:41439
09:03:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/data
09:03:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/wal
09:03:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:03:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.125.62:46583,127.1.125.61:32775,127.1.125.60:44533
09:03:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab
09:03:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.125.62
09:03:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.125.62:46583
09:03:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37653/jwks.json
09:03:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:03:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:03:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/data/info.pb
09:03:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.125.62
09:03:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:03:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:03:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:20.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:20.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:20.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Nov 2026 08:43:21 UTC on e7f111948823
09:03:20.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14810
09:03:20.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:20.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:20.546415 1557 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:20.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:20.547761 1557 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:20.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:20.558975 1563 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:21.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:20.560083 1565 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:21.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:21.816466 1567 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:21.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:21.818724 1566 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1254 milliseconds
09:03:21.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:21.818809 1557 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:03:21.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:21.819896 1557 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:21.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:21.822328 1557 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:21.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:21.823657 1557 hybrid_clock.cc:648] HybridClock initialized: now 1794819801823632 us; error 54 us; skew 500 ppm
09:03:21.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:21.824622 1557 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:21.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:21 dist-test-slave-sqmp krb5kdc[1536](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819801, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:21.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:21.830358 1557 init.cc:377] Logged in from keytab as kudu/127.1.125.62@KRBTEST.COM (short username kudu)
09:03:21.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:21.836915 1557 webserver.cc:577] Webserver started at http://127.1.125.62:34751/ using document root <none> and password file <none>
09:03:21.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:21.837878 1557 fs_manager.cc:362] Metadata directory not provided
09:03:21.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:21.838055 1557 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:21.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:21.838433 1557 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:03:21.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:21.843031 1557 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/data/instance:
09:03:21.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7654450ec8924049aeac8ec73b3d4d54"
09:03:21.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:21 on dist-test-slave-sqmp"
09:03:21.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:21.844254 1557 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/wal/instance:
09:03:21.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7654450ec8924049aeac8ec73b3d4d54"
09:03:21.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:21 on dist-test-slave-sqmp"
09:03:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:21.852443 1557 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
09:03:21.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:21.857932 1576 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:03:21.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:21.859097 1557 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.002s
09:03:21.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:21.859367 1557 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/data,/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/wal
09:03:21.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7654450ec8924049aeac8ec73b3d4d54"
09:03:21.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:21 on dist-test-slave-sqmp"
09:03:21.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:21.859637 1557 fs_report.cc:389] FS layout report
09:03:21.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:21.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/wal
09:03:21.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/wal
09:03:21.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/data/data
09:03:21.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:21.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:21.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:21.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:21.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:21.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:21.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:21.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:21.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:21.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:21.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:21.925915 1557 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:03:21.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:21.932054 1557 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:21.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:21.932569 1557 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:22.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:22.005770 1557 rpc_server.cc:307] RPC server started. Bound to: 127.1.125.62:46583
09:03:22.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:22.005808 1627 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.125.62:46583 every 8 connection(s)
09:03:22.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:22.008639 1557 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/data/info.pb
09:03:22.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:22.014541 1628 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:22.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:22.016882 1524 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu as pid 1557
09:03:22.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:22.017253 1524 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/wal/instance
09:03:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:22.025089 1628 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } has no permanent_uuid. Determining permanent_uuid...
09:03:22.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.125.61@KRBTEST.COM; defaulting to no policy
09:03:22.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:22.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.125.61@KRBTEST.COM" created.
09:03:22.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:22 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819801, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.62@KRBTEST.COM for kudu/127.1.125.62@KRBTEST.COM
09:03:22.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:22.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.125.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:22.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.125.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:22.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:22.064697 1628 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } has no permanent_uuid. Determining permanent_uuid...
09:03:22.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:22.067595 1579 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.125.61:32775: connect: Connection refused (error 111)
09:03:22.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:22.071730 1628 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.125.61:32775: Network error: Client connection negotiation failed: client connection to 127.1.125.61:32775: connect: Connection refused (error 111)
09:03:22.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.1.125.61@KRBTEST.COM; defaulting to no policy
09:03:22.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:22.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.1.125.61@KRBTEST.COM" created.
09:03:22.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:22.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.125.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:22.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.125.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:22.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:22.099539 1524 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/wal
09:03:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/data
09:03:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/logs
09:03:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/data/info.pb
09:03:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:03:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.125.61:32775
09:03:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.125.61
09:03:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab
09:03:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.125.61
09:03:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:03:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:03:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:03:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.125.20:41439
09:03:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:22.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:22.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:22.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.125.62:46583,127.1.125.61:32775,127.1.125.60:44533
09:03:22.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:22.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37653/jwks.json
09:03:22.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
09:03:22.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:03:22.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:22.121248 1628 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } attempt: 1
09:03:22.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:22.126091 1628 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.125.61:32775: Network error: Client connection negotiation failed: client connection to 127.1.125.61:32775: connect: Connection refused (error 111)
09:03:22.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:22.194494 1628 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } attempt: 2
09:03:22.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:22.199510 1628 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.125.61:32775: Network error: Client connection negotiation failed: client connection to 127.1.125.61:32775: connect: Connection refused (error 111)
09:03:22.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:22.291003 1628 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } attempt: 3
09:03:22.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:22.295464 1628 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.125.61:32775: Network error: Client connection negotiation failed: client connection to 127.1.125.61:32775: connect: Connection refused (error 111)
09:03:22.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:22.419482 1637 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:22.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:22.420178 1637 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:22.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:22.420928 1637 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:22.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:22.438839 1628 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } attempt: 4
09:03:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:22.443104 1628 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.125.61:32775: Network error: Client connection negotiation failed: client connection to 127.1.125.61:32775: connect: Connection refused (error 111)
09:03:22.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:22.458492 1637 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
09:03:22.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:22.458807 1637 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
09:03:22.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:22.459286 1637 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:22.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:22.459648 1637 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
09:03:22.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:22.459985 1637 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
09:03:22.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:22.460315 1637 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
09:03:22.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:22.460589 1637 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37653/jwks.json
09:03:22.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:22.501024 1637 master_runner.cc:387] Master server non-default flags:
09:03:22.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/logs
09:03:22.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:22.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:22.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:22.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.125.20:41439
09:03:22.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:22.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:22.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/data
09:03:22.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/wal
09:03:22.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:03:22.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:22.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.125.62:46583,127.1.125.61:32775,127.1.125.60:44533
09:03:22.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab
09:03:22.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.125.61
09:03:22.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:22.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:22.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:22.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.125.61:32775
09:03:22.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:22.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:22.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:22.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37653/jwks.json
09:03:22.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:22.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:03:22.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:03:22.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:22.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:22.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/data/info.pb
09:03:22.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.125.61
09:03:22.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:22.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:22.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:22.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:22.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:22.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:22.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:03:22.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:03:22.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:22.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:22.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:22.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Nov 2026 08:43:21 UTC on e7f111948823
09:03:22.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14810
09:03:22.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:22.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:22.503712 1637 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:22.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:22.504817 1637 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:22.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:22.515940 1645 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:22.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:22.742667 1628 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } attempt: 5
09:03:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:22.752869 1628 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.125.61:32775: Network error: Client connection negotiation failed: client connection to 127.1.125.61:32775: connect: Connection refused (error 111)
09:03:23.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:23.300380 1628 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } attempt: 6
09:03:23.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:23.321177 1628 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.125.61:32775: Network error: Client connection negotiation failed: client connection to 127.1.125.61:32775: connect: Connection refused (error 111)
09:03:24.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.382064 1628 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } attempt: 7
09:03:24.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:22.516170 1646 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:24.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:23.920572 1644 debug-util.cc:398] Leaking SignalData structure 0x7b0800037780 after lost signal to thread 1637
09:03:24.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:24.382660 1637 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.867s user 0.693s sys 1.144s
09:03:24.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:24.384958 1637 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.869s user 0.693s sys 1.144s
09:03:24.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:24.385288 1648 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:24.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:24.388880 1647 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1867 milliseconds
09:03:24.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.388878 1637 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:03:24.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.390288 1637 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:24.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.393245 1637 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:24.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:24.393855 1628 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.125.61:32775: Network error: Client connection negotiation failed: client connection to 127.1.125.61:32775: connect: Connection refused (error 111)
09:03:24.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.394687 1637 hybrid_clock.cc:648] HybridClock initialized: now 1794819804394610 us; error 74 us; skew 500 ppm
09:03:24.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.395586 1637 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:24.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:24 dist-test-slave-sqmp krb5kdc[1536](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819804, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:24.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.401515 1637 init.cc:377] Logged in from keytab as kudu/127.1.125.61@KRBTEST.COM (short username kudu)
09:03:24.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.406960 1637 webserver.cc:577] Webserver started at http://127.1.125.61:36861/ using document root <none> and password file <none>
09:03:24.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.408037 1637 fs_manager.cc:362] Metadata directory not provided
09:03:24.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.408278 1637 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:24.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.408838 1637 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.413188 1637 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/data/instance:
09:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f0b1f437db54e37914014550958bc96"
09:03:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:24 on dist-test-slave-sqmp"
09:03:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.414395 1637 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/wal/instance:
09:03:24.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f0b1f437db54e37914014550958bc96"
09:03:24.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:24 on dist-test-slave-sqmp"
09:03:24.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.422346 1637 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.002s
09:03:24.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.427781 1661 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:03:24.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.428843 1637 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.005s
09:03:24.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.429215 1637 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/data,/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/wal
09:03:24.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f0b1f437db54e37914014550958bc96"
09:03:24.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:24 on dist-test-slave-sqmp"
09:03:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.429603 1637 fs_report.cc:389] FS layout report
09:03:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/wal
09:03:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/wal
09:03:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/data/data
09:03:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:24.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:24.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.499943 1637 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:03:24.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.510236 1637 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:24.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.510633 1637 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:24.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.583957 1637 rpc_server.cc:307] RPC server started. Bound to: 127.1.125.61:32775
09:03:24.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.584025 1713 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.125.61:32775 every 8 connection(s)
09:03:24.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.588317 1637 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/data/info.pb
09:03:24.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.590862 1524 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu as pid 1637
09:03:24.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.591378 1524 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/wal/instance
09:03:24.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.596450 1715 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:24.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.125.60@KRBTEST.COM; defaulting to no policy
09:03:24.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.611332 1715 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } has no permanent_uuid. Determining permanent_uuid...
09:03:24.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:24.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.125.60@KRBTEST.COM" created.
09:03:24.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:24 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819804, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.61@KRBTEST.COM for kudu/127.1.125.62@KRBTEST.COM
09:03:24.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:24.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.125.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:24.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.125.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:24.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.646463 1715 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } has no permanent_uuid. Determining permanent_uuid...
09:03:24.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.1.125.60@KRBTEST.COM; defaulting to no policy
09:03:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:24.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.1.125.60@KRBTEST.COM" created.
09:03:24.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:24 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819804, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.61@KRBTEST.COM for kudu/127.1.125.61@KRBTEST.COM
09:03:24.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.671348 1715 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } has no permanent_uuid. Determining permanent_uuid...
09:03:24.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:24.674089 1665 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.125.60:44533: connect: Connection refused (error 111)
09:03:24.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:24.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.125.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:24.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.125.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.677822 1524 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/wal
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/data
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/logs
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/data/info.pb
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.125.60:44533
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.125.60
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.125.60
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:03:24.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:03:24.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:03:24.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.125.20:41439
09:03:24.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:24.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:24.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:24.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:24.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.125.62:46583,127.1.125.61:32775,127.1.125.60:44533
09:03:24.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:24.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37653/jwks.json
09:03:24.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
09:03:24.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:03:24.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:24.678148 1715 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.125.60:44533: Network error: Client connection negotiation failed: client connection to 127.1.125.60:44533: connect: Connection refused (error 111)
09:03:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.727730 1715 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } attempt: 1
09:03:24.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:24.732072 1715 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.125.60:44533: Network error: Client connection negotiation failed: client connection to 127.1.125.60:44533: connect: Connection refused (error 111)
09:03:24.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.800534 1715 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } attempt: 2
09:03:24.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:24.804904 1715 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.125.60:44533: Network error: Client connection negotiation failed: client connection to 127.1.125.60:44533: connect: Connection refused (error 111)
09:03:24.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:24.896386 1715 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } attempt: 3
09:03:24.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:24.900625 1715 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.125.60:44533: Network error: Client connection negotiation failed: client connection to 127.1.125.60:44533: connect: Connection refused (error 111)
09:03:25.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:25.018687 1725 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:25.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:25.019243 1725 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:25.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:25.019865 1725 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:25.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:25.044047 1715 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } attempt: 4
09:03:25.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:25.048641 1715 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.125.60:44533: Network error: Client connection negotiation failed: client connection to 127.1.125.60:44533: connect: Connection refused (error 111)
09:03:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:25.056780 1725 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
09:03:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:25.057017 1725 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
09:03:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:25.057312 1725 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:25.057610 1725 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
09:03:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:25.057833 1725 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
09:03:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:25.058037 1725 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
09:03:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:25.058192 1725 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37653/jwks.json
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:25.119782 1725 master_runner.cc:387] Master server non-default flags:
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/logs
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.125.20:41439
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/data
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/wal
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.125.62:46583,127.1.125.61:32775,127.1.125.60:44533
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.125.60
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.125.60:44533
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37653/jwks.json
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/data/info.pb
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.125.60
09:03:25.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:25.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:25.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:25.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:25.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:25.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:25.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:03:25.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:03:25.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:25.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:25.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:25.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Nov 2026 08:43:21 UTC on e7f111948823
09:03:25.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14810
09:03:25.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:25.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:25.121109 1725 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:25.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:25.122527 1725 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:25.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:25.135677 1732 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:25.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:25.348076 1715 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } attempt: 5
09:03:25.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:25.358184 1715 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.125.60:44533: Network error: Client connection negotiation failed: client connection to 127.1.125.60:44533: connect: Connection refused (error 111)
09:03:25.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:25.905725 1715 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } attempt: 6
09:03:25.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:25.933574 1715 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.125.60:44533: Network error: Client connection negotiation failed: client connection to 127.1.125.60:44533: connect: Connection refused (error 111)
09:03:26.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.484226 1628 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } attempt: 8
09:03:26.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:26 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819801, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.62@KRBTEST.COM for kudu/127.1.125.61@KRBTEST.COM
09:03:26.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.535127 1628 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } has no permanent_uuid. Determining permanent_uuid...
09:03:26.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:26.558138 1628 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.125.60:44533: Network error: Client connection negotiation failed: client connection to 127.1.125.60:44533: connect: Connection refused (error 111)
09:03:26.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.623692 1628 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } attempt: 1
09:03:26.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:26.630503 1628 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.125.60:44533: Network error: Client connection negotiation failed: client connection to 127.1.125.60:44533: connect: Connection refused (error 111)
09:03:26.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:26.540443 1731 debug-util.cc:398] Leaking SignalData structure 0x7b0800037780 after lost signal to thread 1725
09:03:26.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:26.640863 1731 kernel_stack_watchdog.cc:198] Thread 1725 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 400ms:
09:03:26.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
09:03:26.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
09:03:26.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
09:03:26.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
09:03:26.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:25.137810 1733 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:26.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:26.641726 1725 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.505s user 0.477s sys 0.973s
09:03:26.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:26.642108 1725 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.505s user 0.478s sys 0.974s
09:03:26.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:26.643342 1735 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:26.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:26.646245 1734 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1503 milliseconds
09:03:26.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.646327 1725 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:03:26.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.647598 1725 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:26.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.650131 1725 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:26.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.651496 1725 hybrid_clock.cc:648] HybridClock initialized: now 1794819806651455 us; error 40 us; skew 500 ppm
09:03:26.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.652436 1725 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:26.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:26 dist-test-slave-sqmp krb5kdc[1536](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819806, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:26.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.658212 1725 init.cc:377] Logged in from keytab as kudu/127.1.125.60@KRBTEST.COM (short username kudu)
09:03:26.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.663389 1725 webserver.cc:577] Webserver started at http://127.1.125.60:42205/ using document root <none> and password file <none>
09:03:26.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.664383 1725 fs_manager.cc:362] Metadata directory not provided
09:03:26.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.664633 1725 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:26.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.665184 1725 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:03:26.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.669560 1725 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/data/instance:
09:03:26.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1831907af1cb44e696c849cd58704452"
09:03:26.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:26 on dist-test-slave-sqmp"
09:03:26.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.670755 1725 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/wal/instance:
09:03:26.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1831907af1cb44e696c849cd58704452"
09:03:26.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:26 on dist-test-slave-sqmp"
09:03:26.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.678743 1725 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
09:03:26.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.683991 1628 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } attempt: 2
09:03:26.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.684327 1749 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:03:26.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.685777 1725 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
09:03:26.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.686092 1725 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/data,/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/wal
09:03:26.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1831907af1cb44e696c849cd58704452"
09:03:26.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:26 on dist-test-slave-sqmp"
09:03:26.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.686602 1725 fs_report.cc:389] FS layout report
09:03:26.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:26.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/wal
09:03:26.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/wal
09:03:26.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/data/data
09:03:26.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:26.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:26.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:26.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:26.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:26.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:26.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:26.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:26.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:26.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:26.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:26.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:26.688575 1628 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.125.60:44533: Network error: Client connection negotiation failed: client connection to 127.1.125.60:44533: connect: Connection refused (error 111)
09:03:26.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.751292 1725 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:03:26.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.756803 1725 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:26.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.757194 1725 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:26.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.764945 1628 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } attempt: 3
09:03:26.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:26.769305 1628 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.125.60:44533: Network error: Client connection negotiation failed: client connection to 127.1.125.60:44533: connect: Connection refused (error 111)
09:03:26.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.835096 1725 rpc_server.cc:307] RPC server started. Bound to: 127.1.125.60:44533
09:03:26.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.835281 1800 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.125.60:44533 every 8 connection(s)
09:03:26.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.838272 1725 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/data/info.pb
09:03:26.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.841403 1524 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu as pid 1725
09:03:26.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.842020 1524 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/wal/instance
09:03:26.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.846704 1801 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:26.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.865379 1801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } has no permanent_uuid. Determining permanent_uuid...
09:03:26.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:26 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819800, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.125.62@KRBTEST.COM
09:03:26.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:26 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819806, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.60@KRBTEST.COM for kudu/127.1.125.62@KRBTEST.COM
09:03:26.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:26 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819800, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.125.61@KRBTEST.COM
09:03:26.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.919279 1801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } has no permanent_uuid. Determining permanent_uuid...
09:03:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.924789 1628 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } attempt: 4
09:03:26.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:26 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819806, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.60@KRBTEST.COM for kudu/127.1.125.61@KRBTEST.COM
09:03:26.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:26 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819800, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.125.60@KRBTEST.COM
09:03:26.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:26 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819801, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.62@KRBTEST.COM for kudu/127.1.125.60@KRBTEST.COM
09:03:26.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.961150 1801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } has no permanent_uuid. Determining permanent_uuid...
09:03:26.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:26 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819806, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.60@KRBTEST.COM for kudu/127.1.125.60@KRBTEST.COM
09:03:26.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.981534 1628 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54: Bootstrap starting.
09:03:26.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.988703 1628 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54: Neither blocks nor log segments found. Creating new log.
09:03:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.991400 1628 log.cc:826] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54: Log is configured to *not* fsync() on all Append() calls
09:03:26.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.994161 1715 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } attempt: 7
09:03:26.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:26.998932 1628 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54: No bootstrap required, opened a new log
09:03:27.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.004657 1801 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452: Bootstrap starting.
09:03:27.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:27 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819804, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.61@KRBTEST.COM for kudu/127.1.125.60@KRBTEST.COM
09:03:27.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.013820 1801 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452: Neither blocks nor log segments found. Creating new log.
09:03:27.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.016666 1801 log.cc:826] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452: Log is configured to *not* fsync() on all Append() calls
09:03:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.026584 1628 raft_consensus.cc:360] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } }
09:03:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.027808 1628 raft_consensus.cc:386] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.028278 1628 raft_consensus.cc:748] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7654450ec8924049aeac8ec73b3d4d54, State: Initialized, Role: FOLLOWER
09:03:27.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.029448 1628 consensus_queue.cc:261] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } }
09:03:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.031948 1801 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452: No bootstrap required, opened a new log
09:03:27.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.038210 1813 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } } }
09:03:27.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.039556 1715 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96: Bootstrap starting.
09:03:27.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.042822 1813 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [sys.catalog]: This master's current role is: FOLLOWER
09:03:27.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.042441 1628 sys_catalog.cc:565] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [sys.catalog]: configured and running, proceeding with master startup.
09:03:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.050539 1715 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96: Neither blocks nor log segments found. Creating new log.
09:03:27.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.053249 1715 log.cc:826] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96: Log is configured to *not* fsync() on all Append() calls
09:03:27.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.084914 1715 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96: No bootstrap required, opened a new log
09:03:27.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.071566 1801 raft_consensus.cc:360] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } }
09:03:27.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.091053 1801 raft_consensus.cc:386] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:27.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.091359 1801 raft_consensus.cc:748] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1831907af1cb44e696c849cd58704452, State: Initialized, Role: FOLLOWER
09:03:27.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.098101 1801 consensus_queue.cc:261] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } }
09:03:27.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.108953 1822 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } } }
09:03:27.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.109835 1822 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [sys.catalog]: This master's current role is: FOLLOWER
09:03:27.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.112825 1801 sys_catalog.cc:565] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [sys.catalog]: configured and running, proceeding with master startup.
09:03:27.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.120061 1715 raft_consensus.cc:360] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } }
09:03:27.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.121218 1715 raft_consensus.cc:386] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.121697 1715 raft_consensus.cc:748] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6f0b1f437db54e37914014550958bc96, State: Initialized, Role: FOLLOWER
09:03:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.122920 1715 consensus_queue.cc:261] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } }
09:03:27.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.131508 1830 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } } }
09:03:27.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.135474 1830 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [sys.catalog]: This master's current role is: FOLLOWER
09:03:27.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.135136 1715 sys_catalog.cc:565] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [sys.catalog]: configured and running, proceeding with master startup.
09:03:27.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:27.194243 1582 tablet.cc:2441] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:03:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:27.184724 1828 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:03:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:27.196926 1828 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:03:27.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.207011 1847 catalog_manager.cc:820] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
09:03:27.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.136551 1557 master_runner.cc:187] Error getting master registration for 127.1.125.61:32775: OK, instance_id { permanent_uuid: "6f0b1f437db54e37914014550958bc96" instance_seqno: 1794819804564685 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:03:27.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.237566 1557 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
09:03:27.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:27.253602 1851 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:03:27.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:27.253980 1851 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:03:27.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.260535 1725 master_runner.cc:264] No leader master found from master 1831907af1cb44e696c849cd58704452
09:03:27.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.260861 1725 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
09:03:27.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:27.262614 1755 tablet.cc:2441] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:03:27.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:27.270344 1667 tablet.cc:2441] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:03:27.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.289417 1822 raft_consensus.cc:499] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:03:27.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.289987 1822 raft_consensus.cc:521] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } }
09:03:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.293704 1603 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1831907af1cb44e696c849cd58704452" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7654450ec8924049aeac8ec73b3d4d54" is_pre_election: true
09:03:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.294548 1603 raft_consensus.cc:2490] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1831907af1cb44e696c849cd58704452 in term 0.
09:03:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.294703 1689 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1831907af1cb44e696c849cd58704452" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f0b1f437db54e37914014550958bc96" is_pre_election: true
09:03:27.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.295516 1689 raft_consensus.cc:2490] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1831907af1cb44e696c849cd58704452 in term 0.
09:03:27.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.297132 1752 leader_election.cc:304] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1831907af1cb44e696c849cd58704452, 6f0b1f437db54e37914014550958bc96; no voters:
09:03:27.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.227633 1637 master_runner.cc:187] Error getting master registration for 127.1.125.60:44533: OK, instance_id { permanent_uuid: "1831907af1cb44e696c849cd58704452" instance_seqno: 1794819806812335 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:03:27.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.297883 1637 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
09:03:27.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.299034 1822 leader_election.cc:290] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7654450ec8924049aeac8ec73b3d4d54 (127.1.125.62:46583), 6f0b1f437db54e37914014550958bc96 (127.1.125.61:32775)
09:03:27.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.299723 1822 raft_consensus.cc:2826] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 0 FOLLOWER]: Leader pre-election won for term 1
09:03:27.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.300231 1822 raft_consensus.cc:499] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:03:27.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.300712 1822 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 0 FOLLOWER]: Advancing to term 1
09:03:27.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.307826 1822 raft_consensus.cc:521] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } }
09:03:27.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.310858 1603 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1831907af1cb44e696c849cd58704452" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7654450ec8924049aeac8ec73b3d4d54"
09:03:27.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.311396 1603 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [term 0 FOLLOWER]: Advancing to term 1
09:03:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.319579 1603 raft_consensus.cc:2490] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1831907af1cb44e696c849cd58704452 in term 1.
09:03:27.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.321995 1750 leader_election.cc:304] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1831907af1cb44e696c849cd58704452, 7654450ec8924049aeac8ec73b3d4d54; no voters:
09:03:27.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.323487 1822 leader_election.cc:290] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [CANDIDATE]: Term 1 election: Requested vote from peers 7654450ec8924049aeac8ec73b3d4d54 (127.1.125.62:46583), 6f0b1f437db54e37914014550958bc96 (127.1.125.61:32775)
09:03:27.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.324581 1689 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1831907af1cb44e696c849cd58704452" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6f0b1f437db54e37914014550958bc96"
09:03:27.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.324815 1853 raft_consensus.cc:2826] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 1 FOLLOWER]: Leader election won for term 1
09:03:27.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.325174 1689 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [term 0 FOLLOWER]: Advancing to term 1
09:03:27.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.325613 1853 raft_consensus.cc:705] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 1 LEADER]: Becoming Leader. State: Replica: 1831907af1cb44e696c849cd58704452, State: Running, Role: LEADER
09:03:27.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.326721 1853 consensus_queue.cc:238] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } }
09:03:27.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.332921 1689 raft_consensus.cc:2490] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1831907af1cb44e696c849cd58704452 in term 1.
09:03:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.340519 1822 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1831907af1cb44e696c849cd58704452. Latest consensus state: current_term: 1 leader_uuid: "1831907af1cb44e696c849cd58704452" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } } }
09:03:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.341291 1822 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [sys.catalog]: This master's current role is: LEADER
09:03:27.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.355679 1822 consensus_queue.cc:1049] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:03:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.350728 1855 catalog_manager.cc:1526] Loading table and tablet metadata into memory...
09:03:27.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.367275 1856 consensus_queue.cc:1049] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
09:03:27.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.384611 1813 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1831907af1cb44e696c849cd58704452. Latest consensus state: current_term: 1 leader_uuid: "1831907af1cb44e696c849cd58704452" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } } }
09:03:27.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.390754 1813 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [sys.catalog]: This master's current role is: FOLLOWER
09:03:27.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.391917 1855 catalog_manager.cc:1535] Initializing Kudu cluster ID...
09:03:27.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.402222 1830 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1831907af1cb44e696c849cd58704452. Latest consensus state: current_term: 1 leader_uuid: "1831907af1cb44e696c849cd58704452" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } } }
09:03:27.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.404618 1830 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [sys.catalog]: This master's current role is: FOLLOWER
09:03:27.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.412854 1853 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1831907af1cb44e696c849cd58704452" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } } }
09:03:27.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.415069 1853 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [sys.catalog]: This master's current role is: LEADER
09:03:27.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.415100 1822 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1831907af1cb44e696c849cd58704452" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } } }
09:03:27.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.415781 1822 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [sys.catalog]: This master's current role is: LEADER
09:03:27.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.419221 1813 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1831907af1cb44e696c849cd58704452" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } } }
09:03:27.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.419904 1813 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [sys.catalog]: This master's current role is: FOLLOWER
09:03:27.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.426476 1830 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1831907af1cb44e696c849cd58704452" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } } }
09:03:27.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.427151 1830 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [sys.catalog]: This master's current role is: FOLLOWER
09:03:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.470639 1855 catalog_manager.cc:1398] Generated new cluster ID: b7dfa7d4184c44979b88185ff77a7d33
09:03:27.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.470983 1855 catalog_manager.cc:1546] Initializing Kudu internal certificate authority...
09:03:27.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.512324 1855 catalog_manager.cc:1421] Generated new certificate authority record
09:03:27.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.519222 1855 catalog_manager.cc:1555] Loading token signing keys...
09:03:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.551265 1855 catalog_manager.cc:6221] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452: Generated new TSK 0
09:03:27.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.553798 1855 catalog_manager.cc:1565] Initializing in-progress tserver states...
09:03:27.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.125.1@KRBTEST.COM; defaulting to no policy
09:03:27.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:27.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.125.1@KRBTEST.COM" created.
09:03:27.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:27.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.125.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:27.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.125.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.1.125.1@KRBTEST.COM; defaulting to no policy
09:03:27.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:27.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.1.125.1@KRBTEST.COM" created.
09:03:27.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:27.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.125.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:27.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.125.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:27.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:27.694775 1524 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:27.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:27.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-0/wal
09:03:27.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-0/data
09:03:27.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:27.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:27.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-0/logs
09:03:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-0/data/info.pb
09:03:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:03:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.125.1:0
09:03:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.125.1
09:03:27.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.125.1
09:03:27.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:27.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.125.62:46583,127.1.125.61:32775,127.1.125.60:44533
09:03:27.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab
09:03:27.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.125.1
09:03:27.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:27.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:03:27.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:03:27.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:03:27.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.125.20:41439
09:03:27.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:27.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:27.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:27.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:03:28.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:28.203663 1828 catalog_manager.cc:1310] Loaded cluster ID: b7dfa7d4184c44979b88185ff77a7d33
09:03:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:28.204900 1828 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54: loading cluster ID for follower catalog manager: success
09:03:28.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:28.212576 1828 catalog_manager.cc:1625] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54: acquiring CA information for follower catalog manager: success
09:03:28.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:28.216542 1847 catalog_manager.cc:1310] Loaded cluster ID: b7dfa7d4184c44979b88185ff77a7d33
09:03:28.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:28.217094 1847 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96: loading cluster ID for follower catalog manager: success
09:03:28.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:28.231190 1828 catalog_manager.cc:1653] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:03:28.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:28.236774 1847 catalog_manager.cc:1625] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96: acquiring CA information for follower catalog manager: success
09:03:28.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:28.251175 1847 catalog_manager.cc:1653] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:03:28.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:28.266515 1869 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:28.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:28.267174 1869 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:28.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:28.268193 1869 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:28.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:28.334625 1869 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:28.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:28.336370 1869 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.125.1
09:03:28.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:28.410614 1869 tablet_server_runner.cc:78] Tablet server non-default flags:
09:03:28.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-0/logs
09:03:28.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:28.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:28.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:28.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.125.20:41439
09:03:28.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:28.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:28.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-0/data
09:03:28.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-0/wal
09:03:28.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab
09:03:28.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.125.1
09:03:28.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:28.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:28.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.125.1:0
09:03:28.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:28.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:28.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:03:28.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:03:28.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:28.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:28.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-0/data/info.pb
09:03:28.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.125.1
09:03:28.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:28.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.125.62:46583,127.1.125.61:32775,127.1.125.60:44533
09:03:28.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:28.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:28.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:28.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:28.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:28.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.125.1
09:03:28.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:03:28.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:03:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Nov 2026 08:43:21 UTC on e7f111948823
09:03:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14810
09:03:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:28.412570 1869 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:28.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:28.414106 1869 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:28.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:28.454123 1877 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:29.838356 1876 debug-util.cc:398] Leaking SignalData structure 0x7b080002efa0 after lost signal to thread 1869
09:03:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:30.828002 1876 kernel_stack_watchdog.cc:198] Thread 1869 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 389ms:
09:03:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
09:03:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
09:03:30.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
09:03:30.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
09:03:30.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:28.455461 1878 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:30.849654 1869 thread.cc:642] GCE (cloud detector) Time spent creating pthread: real 2.402s user 0.483s sys 0.869s
09:03:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:30.850045 1869 thread.cc:609] GCE (cloud detector) Time spent starting thread: real 2.402s user 0.483s sys 0.869s
09:03:30.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:30.868253 1885 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:30.869282 1869 server_base.cc:1061] running on GCE node
09:03:30.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:30.870658 1869 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:30.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:30.881843 1869 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:30.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:30.885510 1869 hybrid_clock.cc:648] HybridClock initialized: now 1794819810885432 us; error 79 us; skew 500 ppm
09:03:30.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:30.886612 1869 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:30.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:30 dist-test-slave-sqmp krb5kdc[1536](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819810, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:30.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:30.894905 1869 init.cc:377] Logged in from keytab as kudu/127.1.125.1@KRBTEST.COM (short username kudu)
09:03:30.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:30.901842 1869 webserver.cc:577] Webserver started at http://127.1.125.1:33037/ using document root <none> and password file <none>
09:03:30.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:30.903105 1869 fs_manager.cc:362] Metadata directory not provided
09:03:30.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:30.903391 1869 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:30.904000 1869 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:03:30.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:30.910492 1869 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-0/data/instance:
09:03:30.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb06f37839b14613af75efc167f49119"
09:03:30.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:30 on dist-test-slave-sqmp"
09:03:30.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:30.912271 1869 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-0/wal/instance:
09:03:30.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb06f37839b14613af75efc167f49119"
09:03:30.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:30 on dist-test-slave-sqmp"
09:03:30.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:30.923053 1869 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.005s sys 0.005s
09:03:30.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:30.930871 1891 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:03:30.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:30.932407 1869 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.002s
09:03:30.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:30.932789 1869 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-0/data,/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-0/wal
09:03:30.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb06f37839b14613af75efc167f49119"
09:03:30.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:30 on dist-test-slave-sqmp"
09:03:30.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:30.933342 1869 fs_report.cc:389] FS layout report
09:03:30.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:30.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-0/wal
09:03:30.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-0/wal
09:03:30.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-0/data/data
09:03:30.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:30.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:30.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:30.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:30.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:30.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:30.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:30.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:30.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:30.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:30.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:31.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.006603 1869 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:03:31.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.020390 1869 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:31.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.020970 1869 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.031207 1869 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
09:03:31.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.054992 1869 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:03:31.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.055311 1869 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:03:31.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.055624 1869 ts_tablet_manager.cc:662] Registered 0 tablets
09:03:31.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.055840 1869 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:03:31.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.301160 1869 rpc_server.cc:307] RPC server started. Bound to: 127.1.125.1:35491
09:03:31.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.301321 2004 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.125.1:35491 every 8 connection(s)
09:03:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.304080 1869 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-0/data/info.pb
09:03:31.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.309134 1524 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu as pid 1869
09:03:31.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.310006 1524 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-0/wal/instance
09:03:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.125.2@KRBTEST.COM; defaulting to no policy
09:03:31.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:31.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.125.2@KRBTEST.COM" created.
09:03:31.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:31 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819810, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.1@KRBTEST.COM for kudu/127.1.125.62@KRBTEST.COM
09:03:31.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:31 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819810, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.1@KRBTEST.COM for kudu/127.1.125.60@KRBTEST.COM
09:03:31.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:31 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819810, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.1@KRBTEST.COM for kudu/127.1.125.61@KRBTEST.COM
09:03:31.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.421101 2007 heartbeater.cc:345] Connected to a master server at 127.1.125.62:46583
09:03:31.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.421617 2007 heartbeater.cc:462] Registering TS with master...
09:03:31.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.423036 2007 heartbeater.cc:508] Master 127.1.125.62:46583 requested a full tablet report, sending...
09:03:31.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.424795 2005 heartbeater.cc:345] Connected to a master server at 127.1.125.60:44533
09:03:31.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.425158 2005 heartbeater.cc:462] Registering TS with master...
09:03:31.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.426144 2005 heartbeater.cc:508] Master 127.1.125.60:44533 requested a full tablet report, sending...
09:03:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.429414 1593 ts_manager.cc:210] Registered new tserver with Master: cb06f37839b14613af75efc167f49119 (127.1.125.1:35491)
09:03:31.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.430568 2006 heartbeater.cc:345] Connected to a master server at 127.1.125.61:32775
09:03:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.430920 2006 heartbeater.cc:462] Registering TS with master...
09:03:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.431901 2006 heartbeater.cc:508] Master 127.1.125.61:32775 requested a full tablet report, sending...
09:03:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.435154 1678 ts_manager.cc:210] Registered new tserver with Master: cb06f37839b14613af75efc167f49119 (127.1.125.1:35491)
09:03:31.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.437476 1766 ts_manager.cc:210] Registered new tserver with Master: cb06f37839b14613af75efc167f49119 (127.1.125.1:35491)
09:03:31.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.440838 1766 master_service.cc:507] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.1.125.1@KRBTEST.COM'} at 127.1.125.1:55935
09:03:31.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:31.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.125.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:31.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.125.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:31.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:31.444331 1524 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.1.125.2: real 0.107s user 0.006s sys 0.009s
09:03:31.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.1.125.2@KRBTEST.COM; defaulting to no policy
09:03:31.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:31.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.1.125.2@KRBTEST.COM" created.
09:03:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.125.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:31.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.125.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.487912 1524 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-1/wal
09:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-1/data
09:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-1/logs
09:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-1/data/info.pb
09:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.125.2:0
09:03:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.125.2
09:03:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.125.2
09:03:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.125.62:46583,127.1.125.61:32775,127.1.125.60:44533
09:03:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab
09:03:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.125.2
09:03:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:03:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:03:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:03:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.125.20:41439
09:03:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:03:31.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:31.826403 2019 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:31.826941 2019 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:31.827658 2019 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:31.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:31 dist-test-slave-sqmp krb5kdc[1536](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819811, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:31.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.835815 1572 init.cc:295] Successfully reacquired a new kerberos TGT
09:03:31.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:31.865851 2019 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:31.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:31.866986 2019 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.125.2
09:03:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.906731 2019 tablet_server_runner.cc:78] Tablet server non-default flags:
09:03:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-1/logs
09:03:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.125.20:41439
09:03:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-1/data
09:03:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-1/wal
09:03:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab
09:03:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.125.2
09:03:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.125.2:0
09:03:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:31.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:03:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:03:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-1/data/info.pb
09:03:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.125.2
09:03:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.125.62:46583,127.1.125.61:32775,127.1.125.60:44533
09:03:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.125.2
09:03:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:03:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:03:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Nov 2026 08:43:21 UTC on e7f111948823
09:03:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14810
09:03:31.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:31.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.910332 2019 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:31.911964 2019 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:31.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:31.930040 2027 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:32.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:32.444772 2005 heartbeater.cc:500] Master 127.1.125.60:44533 was elected leader, sending a full tablet report...
09:03:32.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:32.970444 1366 debug-util.cc:398] Leaking SignalData structure 0x7b08000adf60 after lost signal to thread 1240
09:03:32.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:32.972262 1366 debug-util.cc:398] Leaking SignalData structure 0x7b08000b90a0 after lost signal to thread 1369
09:03:33.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:33.332940 2026 debug-util.cc:398] Leaking SignalData structure 0x7b080002efa0 after lost signal to thread 2019
09:03:33.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:33.840134 2026 kernel_stack_watchdog.cc:198] Thread 2019 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 398ms:
09:03:33.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
09:03:33.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
09:03:33.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
09:03:33.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
09:03:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:31.931351 2028 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:33.850347 2019 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.919s user 0.003s sys 0.003s
09:03:33.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:33.850742 2019 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.919s user 0.003s sys 0.003s
09:03:33.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:33.853581 2029 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1920 milliseconds
09:03:33.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:33.855618 2030 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:33.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.855687 2019 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:03:33.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.856889 2019 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:33.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.867749 2019 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:33.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.869207 2019 hybrid_clock.cc:648] HybridClock initialized: now 1794819813869159 us; error 46 us; skew 500 ppm
09:03:33.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.870394 2019 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:33.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:33 dist-test-slave-sqmp krb5kdc[1536](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819813, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:33.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.878427 2019 init.cc:377] Logged in from keytab as kudu/127.1.125.2@KRBTEST.COM (short username kudu)
09:03:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.884953 2019 webserver.cc:577] Webserver started at http://127.1.125.2:42915/ using document root <none> and password file <none>
09:03:33.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.886188 2019 fs_manager.cc:362] Metadata directory not provided
09:03:33.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.886467 2019 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:33.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.887041 2019 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:03:33.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.893860 2019 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-1/data/instance:
09:03:33.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2bea71fe56d4dc1945265f77c929f13"
09:03:33.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:33 on dist-test-slave-sqmp"
09:03:33.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.895334 2019 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-1/wal/instance:
09:03:33.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2bea71fe56d4dc1945265f77c929f13"
09:03:33.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:33 on dist-test-slave-sqmp"
09:03:33.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.904922 2019 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.006s sys 0.005s
09:03:33.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.912338 2045 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:03:33.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.913750 2019 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
09:03:33.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.914113 2019 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-1/data,/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-1/wal
09:03:33.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2bea71fe56d4dc1945265f77c929f13"
09:03:33.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:33 on dist-test-slave-sqmp"
09:03:33.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.914471 2019 fs_report.cc:389] FS layout report
09:03:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-1/wal
09:03:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-1/wal
09:03:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-1/data/data
09:03:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:33.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.969964 2019 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:03:33.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.975229 2019 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:33.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.975751 2019 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:33.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.978861 2019 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
09:03:33.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.983891 2019 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:03:33.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.984113 2019 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:03:33.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.984433 2019 ts_tablet_manager.cc:662] Registered 0 tablets
09:03:33.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:33.984624 2019 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:03:34.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:34.196130 2019 rpc_server.cc:307] RPC server started. Bound to: 127.1.125.2:46241
09:03:34.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:34.196246 2158 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.125.2:46241 every 8 connection(s)
09:03:34.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:34.198901 2019 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-1/data/info.pb
09:03:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:34.209057 1524 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu as pid 2019
09:03:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:34.209563 1524 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-1/wal/instance
09:03:34.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.125.3@KRBTEST.COM; defaulting to no policy
09:03:34.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:34.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.125.3@KRBTEST.COM" created.
09:03:34.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:34 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819813, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.2@KRBTEST.COM for kudu/127.1.125.61@KRBTEST.COM
09:03:34.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:34 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819813, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.2@KRBTEST.COM for kudu/127.1.125.60@KRBTEST.COM
09:03:34.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:34 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819813, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.2@KRBTEST.COM for kudu/127.1.125.62@KRBTEST.COM
09:03:34.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:34.334838 2159 heartbeater.cc:345] Connected to a master server at 127.1.125.60:44533
09:03:34.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:34.335340 2159 heartbeater.cc:462] Registering TS with master...
09:03:34.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:34.336700 2159 heartbeater.cc:508] Master 127.1.125.60:44533 requested a full tablet report, sending...
09:03:34.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:34.338565 2161 heartbeater.cc:345] Connected to a master server at 127.1.125.62:46583
09:03:34.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:34.338922 2161 heartbeater.cc:462] Registering TS with master...
09:03:34.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:34.339607 2160 heartbeater.cc:345] Connected to a master server at 127.1.125.61:32775
09:03:34.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:34.339862 2161 heartbeater.cc:508] Master 127.1.125.62:46583 requested a full tablet report, sending...
09:03:34.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:34.339969 2160 heartbeater.cc:462] Registering TS with master...
09:03:34.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:34.340930 2160 heartbeater.cc:508] Master 127.1.125.61:32775 requested a full tablet report, sending...
09:03:34.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:34.341024 1766 ts_manager.cc:210] Registered new tserver with Master: d2bea71fe56d4dc1945265f77c929f13 (127.1.125.2:46241)
09:03:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:34.342880 1766 master_service.cc:507] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.1.125.2@KRBTEST.COM'} at 127.1.125.2:47759
09:03:34.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:34.343989 1593 ts_manager.cc:210] Registered new tserver with Master: d2bea71fe56d4dc1945265f77c929f13 (127.1.125.2:46241)
09:03:34.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:34.346683 1678 ts_manager.cc:210] Registered new tserver with Master: d2bea71fe56d4dc1945265f77c929f13 (127.1.125.2:46241)
09:03:34.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:34.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.125.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:34.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.125.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:34.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:34.359740 1524 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.1.125.3: real 0.123s user 0.001s sys 0.013s
09:03:34.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.1.125.3@KRBTEST.COM; defaulting to no policy
09:03:34.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:34.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.1.125.3@KRBTEST.COM" created.
09:03:34.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:34.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.125.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:34.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.1.125.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab.
09:03:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:34.403941 1524 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-2/wal
09:03:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-2/data
09:03:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-2/logs
09:03:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-2/data/info.pb
09:03:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:03:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.125.3:0
09:03:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.125.3
09:03:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.125.3
09:03:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.125.62:46583,127.1.125.61:32775,127.1.125.60:44533
09:03:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab
09:03:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.125.3
09:03:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:03:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:03:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:03:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.125.20:41439
09:03:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:03:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:34 dist-test-slave-sqmp krb5kdc[1536](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819814, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:34.406170 1657 init.cc:295] Successfully reacquired a new kerberos TGT
09:03:34.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:34.756480 2173 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:34.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:34.757004 2173 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:34.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:34.757908 2173 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:34.798797 2173 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:34.799852 2173 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.125.3
09:03:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:34.849030 2173 tablet_server_runner.cc:78] Tablet server non-default flags:
09:03:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-2/logs
09:03:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.125.20:41439
09:03:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-2/data
09:03:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-2/wal
09:03:34.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab
09:03:34.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.125.3
09:03:34.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:34.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:34.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.125.3:0
09:03:34.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:34.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:34.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:03:34.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:03:34.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:34.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:34.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-2/data/info.pb
09:03:34.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.125.3
09:03:34.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.125.62:46583,127.1.125.61:32775,127.1.125.60:44533
09:03:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.125.3
09:03:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:03:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:03:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Nov 2026 08:43:21 UTC on e7f111948823
09:03:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14810
09:03:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:34.850776 2173 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:34.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:34.852110 2173 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:34.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:34.889927 2181 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:35.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:35.346976 2159 heartbeater.cc:500] Master 127.1.125.60:44533 was elected leader, sending a full tablet report...
09:03:36.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:34.898080 2182 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:36.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:36.282963 2180 debug-util.cc:398] Leaking SignalData structure 0x7b080002efa0 after lost signal to thread 2173
09:03:36.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:36.370302 2173 thread.cc:642] GCE (cloud detector) Time spent creating pthread: real 1.474s user 0.525s sys 0.821s
09:03:36.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:36.371529 2173 thread.cc:609] GCE (cloud detector) Time spent starting thread: real 1.475s user 0.525s sys 0.821s
09:03:36.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:36.379899 2187 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:36.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.380539 2173 server_base.cc:1061] running on GCE node
09:03:36.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.381762 2173 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.383682 2173 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:36.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.385034 2173 hybrid_clock.cc:648] HybridClock initialized: now 1794819816384997 us; error 34 us; skew 500 ppm
09:03:36.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.385910 2173 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:36.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:36 dist-test-slave-sqmp krb5kdc[1536](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819816, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:36.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.392570 2173 init.cc:377] Logged in from keytab as kudu/127.1.125.3@KRBTEST.COM (short username kudu)
09:03:36.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.397964 2173 webserver.cc:577] Webserver started at http://127.1.125.3:44209/ using document root <none> and password file <none>
09:03:36.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.398883 2173 fs_manager.cc:362] Metadata directory not provided
09:03:36.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.399111 2173 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:36.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.399611 2173 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:03:36.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.404072 2173 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-2/data/instance:
09:03:36.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5b1ef6ec6b39495b95f48d4d4dfac5c3"
09:03:36.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:36 on dist-test-slave-sqmp"
09:03:36.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.405331 2173 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-2/wal/instance:
09:03:36.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5b1ef6ec6b39495b95f48d4d4dfac5c3"
09:03:36.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:36 on dist-test-slave-sqmp"
09:03:36.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.413324 2173 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.001s sys 0.005s
09:03:36.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.420413 2193 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:03:36.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.421712 2173 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.001s
09:03:36.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.422127 2173 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-2/data,/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-2/wal
09:03:36.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5b1ef6ec6b39495b95f48d4d4dfac5c3"
09:03:36.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:36 on dist-test-slave-sqmp"
09:03:36.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.422433 2173 fs_report.cc:389] FS layout report
09:03:36.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:36.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-2/wal
09:03:36.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-2/wal
09:03:36.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-2/data/data
09:03:36.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:36.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:36.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:36.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:36.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:36.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:36.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:36.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:36.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:36.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:36.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:36.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.498476 2173 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:03:36.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.509454 2173 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:36.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.509840 2173 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:36.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.512305 2173 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
09:03:36.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.516278 2173 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:03:36.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.516461 2173 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:03:36.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.516657 2173 ts_tablet_manager.cc:662] Registered 0 tablets
09:03:36.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.516784 2173 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:03:36.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:36 dist-test-slave-sqmp krb5kdc[1536](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819816, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:36.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.662964 1745 init.cc:295] Successfully reacquired a new kerberos TGT
09:03:36.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.664726 2173 rpc_server.cc:307] RPC server started. Bound to: 127.1.125.3:35609
09:03:36.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.664824 2307 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.125.3:35609 every 8 connection(s)
09:03:36.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.667445 2173 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-2/data/info.pb
09:03:36.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.676128 1524 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu as pid 2173
09:03:36.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.676693 1524 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/ts-2/wal/instance
09:03:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:36 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819816, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.3@KRBTEST.COM for kudu/127.1.125.60@KRBTEST.COM
09:03:36.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:36 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819816, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.3@KRBTEST.COM for kudu/127.1.125.61@KRBTEST.COM
09:03:36.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.750382 2309 heartbeater.cc:345] Connected to a master server at 127.1.125.61:32775
09:03:36.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.750870 2309 heartbeater.cc:462] Registering TS with master...
09:03:36.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.752173 2309 heartbeater.cc:508] Master 127.1.125.61:32775 requested a full tablet report, sending...
09:03:36.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:36 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819816, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.3@KRBTEST.COM for kudu/127.1.125.62@KRBTEST.COM
09:03:36.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.755246 1678 ts_manager.cc:210] Registered new tserver with Master: 5b1ef6ec6b39495b95f48d4d4dfac5c3 (127.1.125.3:35609)
09:03:36.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.765805 2308 heartbeater.cc:345] Connected to a master server at 127.1.125.60:44533
09:03:36.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.766131 2308 heartbeater.cc:462] Registering TS with master...
09:03:36.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.767067 2308 heartbeater.cc:508] Master 127.1.125.60:44533 requested a full tablet report, sending...
09:03:36.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.769408 1766 ts_manager.cc:210] Registered new tserver with Master: 5b1ef6ec6b39495b95f48d4d4dfac5c3 (127.1.125.3:35609)
09:03:36.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.771204 1766 master_service.cc:507] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.1.125.3@KRBTEST.COM'} at 127.1.125.3:52765
09:03:36.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.783414 2310 heartbeater.cc:345] Connected to a master server at 127.1.125.62:46583
09:03:36.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.783711 2310 heartbeater.cc:462] Registering TS with master...
09:03:36.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.784364 2310 heartbeater.cc:508] Master 127.1.125.62:46583 requested a full tablet report, sending...
09:03:36.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.786773 1593 ts_manager.cc:210] Registered new tserver with Master: 5b1ef6ec6b39495b95f48d4d4dfac5c3 (127.1.125.3:35609)
09:03:36.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.799171 1524 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
09:03:36.813 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:03:36.814 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
09:03:36.822 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
09:03:36.823 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
09:03:36.827 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.125.62"
port: 46583
}
bound_http_address {
host: "127.1.125.62"
port: 34751
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.125.61"
port: 32775
}
bound_http_address {
host: "127.1.125.61"
port: 36861
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.125.60"
port: 44533
}
bound_http_address {
host: "127.1.125.60"
port: 42205
}
}
}
09:03:36.828 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
09:03:36.829 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.1.125.1"
port: 35491
}
bound_http_address {
host: "127.1.125.1"
port: 33037
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.1.125.2"
port: 46241
}
bound_http_address {
host: "127.1.125.2"
port: 42915
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.1.125.3"
port: 35609
}
bound_http_address {
host: "127.1.125.3"
port: 44209
}
}
}
09:03:36.829 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
09:03:36.830 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
09:03:36.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:36 dist-test-slave-sqmp krb5kdc[1536](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819816, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:36.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
09:03:36.869 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:03:36.872 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/krb5cc
09:03:36.883 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
09:03:36.884 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/krb5cc
09:03:36.885 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
09:03:36.906 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
09:03:36.907 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
09:03:36.908 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
09:03:36.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:36 dist-test-slave-sqmp krb5kdc[1536](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819816, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:36.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
09:03:36.936 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:03:36.936 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/krb5cc
09:03:36.938 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
09:03:36.938 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.125.60:44533
09:03:36.939 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
09:03:36.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.939999 1524 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu with pid 1725
09:03:36.965 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:03:36.965 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.125.61:32775
09:03:36.966 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
09:03:36.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.966578 1524 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu with pid 1637
09:03:36.992 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:03:36.992 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.125.62:46583
09:03:36.992 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
09:03:36.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:36.993352 1524 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu with pid 1557
09:03:37.028 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:03:37.029 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.125.60:44533
09:03:37.032 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
09:03:37.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:37.033668 1524 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:37.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:37.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/wal
09:03:37.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/data
09:03:37.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:37.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:37.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:37.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:37.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:37.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:37.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/logs
09:03:37.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/data/info.pb
09:03:37.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:37.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:37.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:37.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:37.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:37.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:37.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:37.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:37.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:03:37.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:37.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:37.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:37.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.125.60:44533
09:03:37.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.125.60
09:03:37.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42205
09:03:37.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab
09:03:37.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.125.60
09:03:37.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:37.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:03:37.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:03:37.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:03:37.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.125.20:41439
09:03:37.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:37.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:37.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:37.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:37.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.125.62:46583,127.1.125.61:32775,127.1.125.60:44533
09:03:37.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:37.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37653/jwks.json
09:03:37.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
09:03:37.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:03:37.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:37.366148 2159 heartbeater.cc:648] Failed to heartbeat to 127.1.125.60:44533 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.1.125.60:44533: connect: Connection refused (error 111)
09:03:37.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:37.380985 2322 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:37.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:37.381696 2322 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:37.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:37.382496 2322 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:37.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:37.416838 2322 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
09:03:37.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:37.417143 2322 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
09:03:37.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:37.417630 2322 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:37.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:37.417989 2322 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
09:03:37.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:37.418340 2322 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
09:03:37.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:37.418694 2322 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
09:03:37.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:37.418949 2322 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37653/jwks.json
09:03:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:37.459625 2322 master_runner.cc:387] Master server non-default flags:
09:03:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/logs
09:03:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.125.20:41439
09:03:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/data
09:03:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/wal
09:03:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:03:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.125.62:46583,127.1.125.61:32775,127.1.125.60:44533
09:03:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab
09:03:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.125.60
09:03:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.125.60:44533
09:03:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37653/jwks.json
09:03:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:03:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:03:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/data/info.pb
09:03:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.125.60
09:03:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42205
09:03:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:03:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:03:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Nov 2026 08:43:21 UTC on e7f111948823
09:03:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14810
09:03:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:37.461349 2322 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:37.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:37.462818 2322 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:37.465523 2007 heartbeater.cc:648] Failed to heartbeat to 127.1.125.62:46583 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.1.125.62:46583: connect: Connection refused (error 111)
09:03:37.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:37.474282 2332 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:37.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:37.773332 2309 heartbeater.cc:648] Failed to heartbeat to 127.1.125.61:32775 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.1.125.61:32775: connect: Connection refused (error 111)
09:03:37.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:37.774919 2308 heartbeater.cc:500] Master 127.1.125.60:44533 was elected leader, sending a full tablet report...
09:03:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:38.877780 2331 debug-util.cc:398] Leaking SignalData structure 0x7b0800037780 after lost signal to thread 2322
09:03:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:39.372671 2331 kernel_stack_watchdog.cc:198] Thread 2322 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 400ms:
09:03:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
09:03:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
09:03:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
09:03:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
09:03:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:37.475453 2333 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:39.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:39.374083 2322 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.899s user 0.668s sys 1.203s
09:03:39.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:39.374472 2322 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.900s user 0.668s sys 1.203s
09:03:39.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:39.377454 2334 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1899 milliseconds
09:03:39.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:39.378839 2335 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:39.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.378880 2322 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:03:39.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.380146 2322 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:39.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.382536 2322 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:39.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.383872 2322 hybrid_clock.cc:648] HybridClock initialized: now 1794819819383839 us; error 39 us; skew 500 ppm
09:03:39.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.384845 2322 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:39.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:39 dist-test-slave-sqmp krb5kdc[1536](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819819, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:39.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.391386 2322 init.cc:377] Logged in from keytab as kudu/127.1.125.60@KRBTEST.COM (short username kudu)
09:03:39.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.396803 2322 webserver.cc:577] Webserver started at http://127.1.125.60:42205/ using document root <none> and password file <none>
09:03:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.398159 2322 fs_manager.cc:362] Metadata directory not provided
09:03:39.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.398481 2322 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:39.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.406514 2322 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.004s sys 0.004s
09:03:39.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.410648 2355 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:03:39.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.411659 2322 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
09:03:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.412003 2322 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/data,/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/wal
09:03:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1831907af1cb44e696c849cd58704452"
09:03:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:26 on dist-test-slave-sqmp"
09:03:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.413874 2322 fs_report.cc:389] FS layout report
09:03:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/wal
09:03:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/wal
09:03:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/data/data
09:03:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:39.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:39.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:39.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.472311 2322 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:03:39.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.477967 2322 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:39.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.478471 2322 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:39.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.557425 2322 rpc_server.cc:307] RPC server started. Bound to: 127.1.125.60:44533
09:03:39.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.557508 2408 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.125.60:44533 every 8 connection(s)
09:03:39.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.560240 2322 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-2/data/info.pb
09:03:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.568287 1524 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu as pid 2322
09:03:39.569 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:03:39.569 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.125.61:32775
09:03:39.569 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
09:03:39.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.571411 1524 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:39.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:39.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/wal
09:03:39.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/data
09:03:39.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:39.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:39.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/logs
09:03:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/data/info.pb
09:03:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:03:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.125.61:32775
09:03:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.125.61
09:03:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36861
09:03:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab
09:03:39.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.125.61
09:03:39.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:39.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:03:39.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:03:39.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:03:39.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.125.20:41439
09:03:39.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:39.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:39.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:39.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:39.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.125.62:46583,127.1.125.61:32775,127.1.125.60:44533
09:03:39.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:39.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37653/jwks.json
09:03:39.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
09:03:39.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:03:39.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.572108 2409 sys_catalog.cc:263] Verifying existing consensus state
09:03:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.579931 2409 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452: Bootstrap starting.
09:03:39.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.621088 2409 log.cc:826] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452: Log is configured to *not* fsync() on all Append() calls
09:03:39.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.636794 2409 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:03:39.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.637568 2409 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452: Bootstrap complete.
09:03:39.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.661415 2409 raft_consensus.cc:360] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } }
09:03:39.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.664665 2409 raft_consensus.cc:748] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1831907af1cb44e696c849cd58704452, State: Initialized, Role: FOLLOWER
09:03:39.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.665779 2409 consensus_queue.cc:261] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } }
09:03:39.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.668861 2413 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } } }
09:03:39.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.670143 2413 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [sys.catalog]: This master's current role is: FOLLOWER
09:03:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.670722 2409 sys_catalog.cc:565] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [sys.catalog]: configured and running, proceeding with master startup.
09:03:39.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.721041 2424 catalog_manager.cc:1310] Loaded cluster ID: b7dfa7d4184c44979b88185ff77a7d33
09:03:39.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.721314 2424 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452: loading cluster ID for follower catalog manager: success
09:03:39.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.729636 2424 catalog_manager.cc:1625] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452: acquiring CA information for follower catalog manager: success
09:03:39.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.735172 2424 catalog_manager.cc:1653] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:03:39.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.717188 2322 master_runner.cc:187] Error getting master registration for 127.1.125.62:46583: Network error: Client connection negotiation failed: client connection to 127.1.125.62:46583: connect: Connection refused (error 111),
09:03:39.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.741971 2322 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
09:03:39.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.808072 2308 heartbeater.cc:345] Connected to a master server at 127.1.125.60:44533
09:03:39.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.808604 2308 heartbeater.cc:500] Master 127.1.125.60:44533 was elected leader, sending a full tablet report...
09:03:39.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.814774 2374 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "5b1ef6ec6b39495b95f48d4d4dfac5c3" instance_seqno: 1794819816632158) as {username='kudu', principal='kudu/127.1.125.3@KRBTEST.COM'} at 127.1.125.3:59433; Asking this server to re-register.
09:03:39.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.816419 2308 heartbeater.cc:462] Registering TS with master...
09:03:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:39.819964 2374 ts_manager.cc:210] Registered new tserver with Master: 5b1ef6ec6b39495b95f48d4d4dfac5c3 (127.1.125.3:35609)
09:03:39.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:39.951292 2411 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:39.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:39.952112 2411 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:39.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:39.953285 2411 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:39.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:39.992408 2411 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
09:03:39.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:39.992751 2411 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
09:03:39.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:39.993242 2411 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:39.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:39.993549 2411 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
09:03:39.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:39.993780 2411 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
09:03:39.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:39.993984 2411 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
09:03:39.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:39.994145 2411 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37653/jwks.json
09:03:40.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:40.033814 2411 master_runner.cc:387] Master server non-default flags:
09:03:40.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/logs
09:03:40.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.125.20:41439
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/data
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/wal
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.125.62:46583,127.1.125.61:32775,127.1.125.60:44533
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.125.61
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.125.61:32775
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37653/jwks.json
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:03:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:40.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:40.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/data/info.pb
09:03:40.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.125.61
09:03:40.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36861
09:03:40.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:40.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:40.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:40.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:40.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:40.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:03:40.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:03:40.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:40.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:40.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:40.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Nov 2026 08:43:21 UTC on e7f111948823
09:03:40.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14810
09:03:40.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:40.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:40.035586 2411 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:40.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:40.036978 2411 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:40.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:40.053007 2437 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:40.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:40.500352 2159 heartbeater.cc:345] Connected to a master server at 127.1.125.60:44533
09:03:40.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:40.517175 2374 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "d2bea71fe56d4dc1945265f77c929f13" instance_seqno: 1794819814149103) as {username='kudu', principal='kudu/127.1.125.2@KRBTEST.COM'} at 127.1.125.2:55297; Asking this server to re-register.
09:03:40.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:40.520988 2159 heartbeater.cc:462] Registering TS with master...
09:03:40.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:40.525779 2374 ts_manager.cc:210] Registered new tserver with Master: d2bea71fe56d4dc1945265f77c929f13 (127.1.125.2:46241)
09:03:40.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:40.581305 2005 heartbeater.cc:345] Connected to a master server at 127.1.125.60:44533
09:03:40.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:40.585187 2374 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "cb06f37839b14613af75efc167f49119" instance_seqno: 1794819811241939) as {username='kudu', principal='kudu/127.1.125.1@KRBTEST.COM'} at 127.1.125.1:34417; Asking this server to re-register.
09:03:40.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:40.587668 2005 heartbeater.cc:462] Registering TS with master...
09:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:40.591934 2374 ts_manager.cc:210] Registered new tserver with Master: cb06f37839b14613af75efc167f49119 (127.1.125.1:35491)
09:03:40.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:40 dist-test-slave-sqmp krb5kdc[1536](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819820, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:40.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:40.902132 1887 init.cc:295] Successfully reacquired a new kerberos TGT
09:03:41.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:41.450018 2452 raft_consensus.cc:499] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:03:41.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:41.454310 2452 raft_consensus.cc:521] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } }
09:03:41.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:41.506417 2452 leader_election.cc:290] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 7654450ec8924049aeac8ec73b3d4d54 (127.1.125.62:46583), 6f0b1f437db54e37914014550958bc96 (127.1.125.61:32775)
09:03:41.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:41.523548 2358 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.125.61:32775: connect: Connection refused (error 111)
09:03:41.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:41.538304 2356 leader_election.cc:336] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 7654450ec8924049aeac8ec73b3d4d54 (127.1.125.62:46583): Network error: Client connection negotiation failed: client connection to 127.1.125.62:46583: connect: Connection refused (error 111)
09:03:41.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:41.553331 2358 leader_election.cc:336] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6f0b1f437db54e37914014550958bc96 (127.1.125.61:32775): Network error: Client connection negotiation failed: client connection to 127.1.125.61:32775: connect: Connection refused (error 111)
09:03:41.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:41.553913 2358 leader_election.cc:304] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1831907af1cb44e696c849cd58704452; no voters: 6f0b1f437db54e37914014550958bc96, 7654450ec8924049aeac8ec73b3d4d54
09:03:41.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:41.564576 2452 raft_consensus.cc:2771] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
09:03:41.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:41.456557 2435 debug-util.cc:398] Leaking SignalData structure 0x7b0800037780 after lost signal to thread 2411
09:03:41.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:41.888235 2435 kernel_stack_watchdog.cc:198] Thread 2411 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 400ms:
09:03:41.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
09:03:41.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
09:03:41.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
09:03:41.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
09:03:41.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:41.889745 2438 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1831 milliseconds
09:03:41.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:41.890234 2411 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.837s user 0.000s sys 0.006s
09:03:41.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:40.053013 2436 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:41.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:41.890671 2411 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.838s user 0.000s sys 0.006s
09:03:41.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:41.892643 2411 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:03:41.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:41.892714 2440 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:41.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:41.897163 2411 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:41.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:41.900233 2411 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:41.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:41.903770 2411 hybrid_clock.cc:648] HybridClock initialized: now 1794819821903731 us; error 37 us; skew 500 ppm
09:03:41.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:41.904816 2411 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:41.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:41 dist-test-slave-sqmp krb5kdc[1536](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819821, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:41.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:41.912684 2411 init.cc:377] Logged in from keytab as kudu/127.1.125.61@KRBTEST.COM (short username kudu)
09:03:41.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:41.920892 2411 webserver.cc:577] Webserver started at http://127.1.125.61:36861/ using document root <none> and password file <none>
09:03:41.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:41.922394 2411 fs_manager.cc:362] Metadata directory not provided
09:03:41.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:41.922693 2411 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:41.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:41.933743 2411 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.007s sys 0.000s
09:03:41.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:41.939385 2469 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:03:41.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:41.940688 2411 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
09:03:41.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:41.941074 2411 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/data,/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/wal
09:03:41.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f0b1f437db54e37914014550958bc96"
09:03:41.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:24 on dist-test-slave-sqmp"
09:03:41.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:41.943895 2411 fs_report.cc:389] FS layout report
09:03:41.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:41.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/wal
09:03:41.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/wal
09:03:41.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/data/data
09:03:41.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:41.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:41.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:41.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:41.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:41.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:41.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:41.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:41.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:41.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:41.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:42.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.005980 2411 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:03:42.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.016126 2411 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:42.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.016659 2411 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:42.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.112696 2411 rpc_server.cc:307] RPC server started. Bound to: 127.1.125.61:32775
09:03:42.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.112813 2520 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.125.61:32775 every 8 connection(s)
09:03:42.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.116556 2411 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-1/data/info.pb
09:03:42.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.121037 1524 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu as pid 2411
09:03:42.122 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:03:42.122 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.125.62:46583
09:03:42.122 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
09:03:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.125208 1524 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu
09:03:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/wal
09:03:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/data
09:03:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/logs
09:03:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/data/info.pb
09:03:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.125.62:46583
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.125.62
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34751
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.125.62
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.125.20:41439
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.125.62:46583,127.1.125.61:32775,127.1.125.60:44533
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37653/jwks.json
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
09:03:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:03:42.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.135068 2521 sys_catalog.cc:263] Verifying existing consensus state
09:03:42.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.143780 2521 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96: Bootstrap starting.
09:03:42.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.207585 2521 log.cc:826] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96: Log is configured to *not* fsync() on all Append() calls
09:03:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.236792 2521 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:03:42.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.237975 2521 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96: Bootstrap complete.
09:03:42.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.279083 2521 raft_consensus.cc:360] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } }
09:03:42.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.283129 2521 raft_consensus.cc:748] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6f0b1f437db54e37914014550958bc96, State: Initialized, Role: FOLLOWER
09:03:42.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.284642 2521 consensus_queue.cc:261] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } }
09:03:42.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.300750 2521 sys_catalog.cc:565] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [sys.catalog]: configured and running, proceeding with master startup.
09:03:42.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.314957 2525 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } } }
09:03:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.315873 2525 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [sys.catalog]: This master's current role is: FOLLOWER
09:03:42.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.357736 2536 catalog_manager.cc:1310] Loaded cluster ID: b7dfa7d4184c44979b88185ff77a7d33
09:03:42.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.358024 2536 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96: loading cluster ID for follower catalog manager: success
09:03:42.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.376602 2536 catalog_manager.cc:1625] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96: acquiring CA information for follower catalog manager: success
09:03:42.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.382511 2536 catalog_manager.cc:1653] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:03:42.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.368356 2411 master_runner.cc:187] Error getting master registration for 127.1.125.62:46583: Network error: Client connection negotiation failed: client connection to 127.1.125.62:46583: connect: Connection refused (error 111),
09:03:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.399802 2411 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
09:03:42.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.529682 2160 heartbeater.cc:345] Connected to a master server at 127.1.125.61:32775
09:03:42.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.541851 2486 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "d2bea71fe56d4dc1945265f77c929f13" instance_seqno: 1794819814149103) as {username='kudu', principal='kudu/127.1.125.2@KRBTEST.COM'} at 127.1.125.2:34145; Asking this server to re-register.
09:03:42.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.544484 2160 heartbeater.cc:462] Registering TS with master...
09:03:42.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.556205 2485 ts_manager.cc:210] Registered new tserver with Master: d2bea71fe56d4dc1945265f77c929f13 (127.1.125.2:46241)
09:03:42.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.586249 2006 heartbeater.cc:345] Connected to a master server at 127.1.125.61:32775
09:03:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.589159 2486 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "cb06f37839b14613af75efc167f49119" instance_seqno: 1794819811241939) as {username='kudu', principal='kudu/127.1.125.1@KRBTEST.COM'} at 127.1.125.1:37569; Asking this server to re-register.
09:03:42.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.590498 2006 heartbeater.cc:462] Registering TS with master...
09:03:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.595220 2486 ts_manager.cc:210] Registered new tserver with Master: cb06f37839b14613af75efc167f49119 (127.1.125.1:35491)
09:03:42.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:42.624343 2523 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:42.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:42.625056 2523 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:42.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:42.625916 2523 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:42.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:42.675873 2523 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
09:03:42.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:42.676504 2523 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
09:03:42.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:42.677182 2523 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:42.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:42.677716 2523 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
09:03:42.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:42.678191 2523 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
09:03:42.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:42.678643 2523 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
09:03:42.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:42.679026 2523 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37653/jwks.json
09:03:42.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.721024 2523 master_runner.cc:387] Master server non-default flags:
09:03:42.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/logs
09:03:42.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:42.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:42.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:42.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.125.20:41439
09:03:42.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:42.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:42.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/data
09:03:42.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/wal
09:03:42.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:03:42.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:42.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.125.62:46583,127.1.125.61:32775,127.1.125.60:44533
09:03:42.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/krb5kdc/kudu.keytab
09:03:42.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.125.62
09:03:42.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:42.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:42.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:42.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.125.62:46583
09:03:42.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:42.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:42.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:42.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37653/jwks.json
09:03:42.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:42.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:03:42.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:03:42.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:42.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:42.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/data/info.pb
09:03:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.125.62
09:03:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34751
09:03:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:03:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:03:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Nov 2026 08:43:21 UTC on e7f111948823
09:03:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14810
09:03:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.722757 2523 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.724115 2523 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:42.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:42.754667 2550 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:42.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:42.877907 1224 debug-util.cc:398] Leaking SignalData structure 0x7b08000b4340 after lost signal to thread 1098
09:03:42.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:42.879201 1224 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba540 after lost signal to thread 1227
09:03:42.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.914392 2309 heartbeater.cc:345] Connected to a master server at 127.1.125.61:32775
09:03:42.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.917310 2486 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "5b1ef6ec6b39495b95f48d4d4dfac5c3" instance_seqno: 1794819816632158) as {username='kudu', principal='kudu/127.1.125.3@KRBTEST.COM'} at 127.1.125.3:57763; Asking this server to re-register.
09:03:42.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.920256 2309 heartbeater.cc:462] Registering TS with master...
09:03:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:42.923985 2486 ts_manager.cc:210] Registered new tserver with Master: 5b1ef6ec6b39495b95f48d4d4dfac5c3 (127.1.125.3:35609)
09:03:43.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.271993 2557 raft_consensus.cc:499] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:03:43.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.280048 2557 raft_consensus.cc:521] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } }
09:03:43.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.330722 2557 leader_election.cc:290] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 7654450ec8924049aeac8ec73b3d4d54 (127.1.125.62:46583), 6f0b1f437db54e37914014550958bc96 (127.1.125.61:32775)
09:03:43.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.376961 2496 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1831907af1cb44e696c849cd58704452" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6f0b1f437db54e37914014550958bc96" is_pre_election: true
09:03:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.378480 2496 raft_consensus.cc:2490] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1831907af1cb44e696c849cd58704452 in term 1.
09:03:43.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.381634 2358 leader_election.cc:304] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1831907af1cb44e696c849cd58704452, 6f0b1f437db54e37914014550958bc96; no voters:
09:03:43.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.383216 2557 raft_consensus.cc:2826] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 1 FOLLOWER]: Leader pre-election won for term 2
09:03:43.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.383896 2557 raft_consensus.cc:499] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:03:43.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.384683 2557 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 1 FOLLOWER]: Advancing to term 2
09:03:43.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:43.407994 2356 leader_election.cc:336] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 7654450ec8924049aeac8ec73b3d4d54 (127.1.125.62:46583): Network error: Client connection negotiation failed: client connection to 127.1.125.62:46583: connect: Connection refused (error 111)
09:03:43.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.466249 2557 raft_consensus.cc:521] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } }
09:03:43.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:43.493317 2356 leader_election.cc:336] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 7654450ec8924049aeac8ec73b3d4d54 (127.1.125.62:46583): Network error: Client connection negotiation failed: client connection to 127.1.125.62:46583: connect: Connection refused (error 111)
09:03:43.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.498343 2496 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1831907af1cb44e696c849cd58704452" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6f0b1f437db54e37914014550958bc96"
09:03:43.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.499178 2496 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [term 1 FOLLOWER]: Advancing to term 2
09:03:43.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.519992 2557 leader_election.cc:290] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [CANDIDATE]: Term 2 election: Requested vote from peers 7654450ec8924049aeac8ec73b3d4d54 (127.1.125.62:46583), 6f0b1f437db54e37914014550958bc96 (127.1.125.61:32775)
09:03:43.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.529002 2496 raft_consensus.cc:2490] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1831907af1cb44e696c849cd58704452 in term 2.
09:03:43.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.535827 2358 leader_election.cc:304] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1831907af1cb44e696c849cd58704452, 6f0b1f437db54e37914014550958bc96; no voters: 7654450ec8924049aeac8ec73b3d4d54
09:03:43.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.547130 2557 raft_consensus.cc:2826] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 2 FOLLOWER]: Leader election won for term 2
09:03:43.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.574117 2557 raft_consensus.cc:705] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [term 2 LEADER]: Becoming Leader. State: Replica: 1831907af1cb44e696c849cd58704452, State: Running, Role: LEADER
09:03:43.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.577320 2557 consensus_queue.cc:238] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } }
09:03:43.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.614499 2562 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1831907af1cb44e696c849cd58704452. Latest consensus state: current_term: 2 leader_uuid: "1831907af1cb44e696c849cd58704452" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } } }
09:03:43.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.615262 2562 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [sys.catalog]: This master's current role is: LEADER
09:03:43.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.622876 2565 catalog_manager.cc:1526] Loading table and tablet metadata into memory...
09:03:43.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.629958 2496 raft_consensus.cc:1285] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [term 2 FOLLOWER]: Refusing update from remote peer 1831907af1cb44e696c849cd58704452: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
09:03:43.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.634420 2557 consensus_queue.cc:1049] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
09:03:43.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:43.642549 2356 consensus_peers.cc:597] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 -> Peer 7654450ec8924049aeac8ec73b3d4d54 (127.1.125.62:46583): Couldn't send request to peer 7654450ec8924049aeac8ec73b3d4d54. Status: Network error: Client connection negotiation failed: client connection to 127.1.125.62:46583: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
09:03:43.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.646304 2565 catalog_manager.cc:1535] Initializing Kudu cluster ID...
09:03:43.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.660297 2565 catalog_manager.cc:1310] Loaded cluster ID: b7dfa7d4184c44979b88185ff77a7d33
09:03:43.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.660756 2565 catalog_manager.cc:1546] Initializing Kudu internal certificate authority...
09:03:43.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.666914 2565 catalog_manager.cc:1555] Loading token signing keys...
09:03:43.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.682536 2565 catalog_manager.cc:6232] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452: Loaded TSK: 0
09:03:43.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.684113 2565 catalog_manager.cc:1565] Initializing in-progress tserver states...
09:03:43.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.733161 2562 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "1831907af1cb44e696c849cd58704452" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } } }
09:03:43.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.733817 2562 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [sys.catalog]: This master's current role is: LEADER
09:03:43.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.732549 2566 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1831907af1cb44e696c849cd58704452. Latest consensus state: current_term: 2 leader_uuid: "1831907af1cb44e696c849cd58704452" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } } }
09:03:43.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.748056 2566 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [sys.catalog]: This master's current role is: FOLLOWER
09:03:43.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.747291 2570 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "1831907af1cb44e696c849cd58704452" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } } }
09:03:43.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.755388 2570 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6f0b1f437db54e37914014550958bc96 [sys.catalog]: This master's current role is: FOLLOWER
09:03:43.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:43 dist-test-slave-sqmp krb5kdc[1536](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819823, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:43.883741 2041 init.cc:295] Successfully reacquired a new kerberos TGT
09:03:44.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:44.154771 2549 debug-util.cc:398] Leaking SignalData structure 0x7b0800037780 after lost signal to thread 2523
09:03:44.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:44.590771 2523 thread.cc:642] GCE (cloud detector) Time spent creating pthread: real 1.835s user 0.320s sys 0.485s
09:03:44.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:44.591228 2523 thread.cc:609] GCE (cloud detector) Time spent starting thread: real 1.836s user 0.320s sys 0.485s
09:03:44.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:42.757216 2551 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:44.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.617192 2523 server_base.cc:1061] running on GCE node
09:03:44.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:44.618896 2576 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:44.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.620442 2523 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:44.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.633299 2523 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:44.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.636893 2523 hybrid_clock.cc:648] HybridClock initialized: now 1794819824636845 us; error 40 us; skew 500 ppm
09:03:44.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.638085 2523 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:44.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:44 dist-test-slave-sqmp krb5kdc[1536](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819824, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:44.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.646363 2523 init.cc:377] Logged in from keytab as kudu/127.1.125.62@KRBTEST.COM (short username kudu)
09:03:44.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.652608 2523 webserver.cc:577] Webserver started at http://127.1.125.62:34751/ using document root <none> and password file <none>
09:03:44.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.653903 2523 fs_manager.cc:362] Metadata directory not provided
09:03:44.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.654214 2523 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:44.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.715353 2523 fs_manager.cc:714] Time spent opening directory manager: real 0.057s user 0.038s sys 0.015s
09:03:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.721864 2584 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:03:44.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.723630 2523 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
09:03:44.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.724064 2523 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/data,/tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/wal
09:03:44.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7654450ec8924049aeac8ec73b3d4d54"
09:03:44.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-16 09:03:21 on dist-test-slave-sqmp"
09:03:44.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.727063 2523 fs_report.cc:389] FS layout report
09:03:44.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/wal
09:03:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/wal
09:03:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/data/data
09:03:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:44.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:44.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:44.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:44.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:44.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:44.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.745796 2322 master_runner.cc:187] Error getting master registration for 127.1.125.62:46583: Network error: Client connection negotiation failed: client connection to 127.1.125.62:46583: connect: Connection refused (error 111),
09:03:44.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.746256 2322 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
09:03:44.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.828452 2523 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:03:44.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.836400 2523 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:44.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.836954 2523 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:44.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.855053 2308 heartbeater.cc:500] Master 127.1.125.60:44533 was elected leader, sending a full tablet report...
09:03:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.983619 2523 rpc_server.cc:307] RPC server started. Bound to: 127.1.125.62:46583
09:03:44.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.983698 2638 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.125.62:46583 every 8 connection(s)
09:03:44.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.988037 2523 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskQWUX02/test-tmp/mini-kudu-cluster13504603622043933203/master-0/data/info.pb
09:03:44.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:44.989018 1524 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu as pid 2523
09:03:44.989 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:03:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.010375 2639 sys_catalog.cc:263] Verifying existing consensus state
09:03:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.020188 2639 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54: Bootstrap starting.
09:03:45.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.100080 2639 log.cc:826] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54: Log is configured to *not* fsync() on all Append() calls
09:03:45.107 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.125.60:44533(127.1.125.60:44533)] connecting to peer
09:03:45.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.142913 2639 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:03:45.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.143877 2639 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54: Bootstrap complete.
09:03:45.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:45 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819819, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.60@KRBTEST.COM for kudu/127.1.125.62@KRBTEST.COM
09:03:45.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.217110 2639 raft_consensus.cc:360] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } }
09:03:45.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.220323 2639 raft_consensus.cc:748] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7654450ec8924049aeac8ec73b3d4d54, State: Initialized, Role: FOLLOWER
09:03:45.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.221262 2639 consensus_queue.cc:261] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } }
09:03:45.234 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.125.61:32775(127.1.125.61:32775)] connecting to peer
09:03:45.238 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.125.62:46583(127.1.125.62:46583)] connecting to peer
09:03:45.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.263775 2639 sys_catalog.cc:565] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [sys.catalog]: configured and running, proceeding with master startup.
09:03:45.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.264323 2613 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [term 1 FOLLOWER]: Advancing to term 2
09:03:45.289 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.1.125.62:46583(127.1.125.62:46583)] Successfully connected to peer
09:03:45.294 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.1.125.61:32775(127.1.125.61:32775)] Successfully connected to peer
09:03:45.310 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.1.125.60:44533(127.1.125.60:44533)] Successfully connected to peer
09:03:45.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.503558 2645 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 leader_uuid: "1831907af1cb44e696c849cd58704452" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } } }
09:03:45.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.506029 2645 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [sys.catalog]: This master's current role is: FOLLOWER
09:03:45.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.500473 2654 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1831907af1cb44e696c849cd58704452. Latest consensus state: current_term: 2 leader_uuid: "1831907af1cb44e696c849cd58704452" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } } }
09:03:45.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.506976 2654 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [sys.catalog]: This master's current role is: FOLLOWER
09:03:45.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.525455 2659 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "1831907af1cb44e696c849cd58704452" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } } }
09:03:45.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.528265 2659 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7654450ec8924049aeac8ec73b3d4d54 [sys.catalog]: This master's current role is: FOLLOWER
09:03:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.553721 2159 heartbeater.cc:500] Master 127.1.125.60:44533 was elected leader, sending a full tablet report...
09:03:45.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.568140 2664 catalog_manager.cc:820] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
09:03:45.603 [WARN - kudu-nio-2] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
09:03:45.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:45 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819823, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.2@KRBTEST.COM for kudu/127.1.125.62@KRBTEST.COM
09:03:45.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:45 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819824, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.62@KRBTEST.COM for kudu/127.1.125.62@KRBTEST.COM
09:03:45.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.627040 2005 heartbeater.cc:500] Master 127.1.125.60:44533 was elected leader, sending a full tablet report...
09:03:45.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.642313 2557 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "1831907af1cb44e696c849cd58704452" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7654450ec8924049aeac8ec73b3d4d54" member_type: VOTER last_known_addr { host: "127.1.125.62" port: 46583 } } peers { permanent_uuid: "6f0b1f437db54e37914014550958bc96" member_type: VOTER last_known_addr { host: "127.1.125.61" port: 32775 } } peers { permanent_uuid: "1831907af1cb44e696c849cd58704452" member_type: VOTER last_known_addr { host: "127.1.125.60" port: 44533 } } }
09:03:45.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.643280 2557 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1831907af1cb44e696c849cd58704452 [sys.catalog]: This master's current role is: LEADER
09:03:45.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:45 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819820, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.1@KRBTEST.COM for kudu/127.1.125.62@KRBTEST.COM
09:03:45.675 [DEBUG - kudu-nio-1] (Negotiator.java:528) SASL mechanism GSSAPI chosen for peer 127.1.125.61
09:03:45.677 [DEBUG - kudu-nio-2] (Negotiator.java:528) SASL mechanism GSSAPI chosen for peer 127.1.125.62
09:03:45.681 [DEBUG - kudu-nio-0] (Negotiator.java:528) SASL mechanism GSSAPI chosen for peer 127.1.125.60
09:03:45.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.691701 2161 heartbeater.cc:345] Connected to a master server at 127.1.125.62:46583
09:03:45.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.695019 2007 heartbeater.cc:345] Connected to a master server at 127.1.125.62:46583
09:03:45.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.698179 2602 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "cb06f37839b14613af75efc167f49119" instance_seqno: 1794819811241939) as {username='kudu', principal='kudu/127.1.125.1@KRBTEST.COM'} at 127.1.125.1:58211; Asking this server to re-register.
09:03:45.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.699357 2603 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "d2bea71fe56d4dc1945265f77c929f13" instance_seqno: 1794819814149103) as {username='kudu', principal='kudu/127.1.125.2@KRBTEST.COM'} at 127.1.125.2:39195; Asking this server to re-register.
09:03:45.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.699743 2007 heartbeater.cc:462] Registering TS with master...
09:03:45.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.701570 2161 heartbeater.cc:462] Registering TS with master...
09:03:45.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.705168 2602 ts_manager.cc:210] Registered new tserver with Master: d2bea71fe56d4dc1945265f77c929f13 (127.1.125.2:46241)
09:03:45.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.705323 2603 ts_manager.cc:210] Registered new tserver with Master: cb06f37839b14613af75efc167f49119 (127.1.125.1:35491)
09:03:45.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:45 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819824, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.62@KRBTEST.COM for kudu/127.1.125.61@KRBTEST.COM
09:03:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 16 09:03:45 dist-test-slave-sqmp krb5kdc[1536](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794819824, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.125.62@KRBTEST.COM for kudu/127.1.125.60@KRBTEST.COM
09:03:45.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.992525 2310 heartbeater.cc:345] Connected to a master server at 127.1.125.62:46583
09:03:45.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.995419 2603 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "5b1ef6ec6b39495b95f48d4d4dfac5c3" instance_seqno: 1794819816632158) as {username='kudu', principal='kudu/127.1.125.3@KRBTEST.COM'} at 127.1.125.3:46261; Asking this server to re-register.
09:03:45.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.996738 2310 heartbeater.cc:462] Registering TS with master...
09:03:46.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:45.999727 2603 ts_manager.cc:210] Registered new tserver with Master: 5b1ef6ec6b39495b95f48d4d4dfac5c3 (127.1.125.3:35609)
09:03:46.024 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.125.60:44533(127.1.125.60:44533)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
09:03:46.024 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.1.125.62:46583(127.1.125.62:46583)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
09:03:46.024 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.125.61:32775(127.1.125.61:32775)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
09:03:46.025 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.125.60:44533: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
09:03:46.025 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.1.125.62:46583: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
09:03:46.025 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.125.61:32775: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
09:03:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:46.027861 2653 negotiation.cc:336] Failed RPC negotiation. Trace:
09:03:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.309395 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:43580 (local address 127.1.125.60:44533)
09:03:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.332078 (+ 22683us) server_negotiation.cc:207] Beginning negotiation
09:03:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.332109 (+ 31us) server_negotiation.cc:400] Waiting for connection header
09:03:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.427226 (+ 95117us) server_negotiation.cc:408] Connection header received
09:03:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.582170 (+154944us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
09:03:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.582203 (+ 33us) server_negotiation.cc:462] Received NEGOTIATE request from client
09:03:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.582443 (+ 240us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
09:03:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.582628 (+ 185us) server_negotiation.cc:227] Negotiated authn=SASL
09:03:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.903793 (+321165us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
09:03:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.905272 (+ 1479us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
09:03:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:46.017778 (+112506us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
09:03:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:46.018412 (+ 634us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
09:03:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:46.027804 (+ 9392us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:43580: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:43580): Cannot send after transport endpoint shutdown (error 108)
09:03:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":22109,"thread_start_us":22676,"threads_started":1}
09:03:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:46.028911 2643 negotiation.cc:336] Failed RPC negotiation. Trace:
09:03:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.305649 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:49784 (local address 127.1.125.62:46583)
09:03:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.306600 (+ 951us) server_negotiation.cc:207] Beginning negotiation
09:03:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.306637 (+ 37us) server_negotiation.cc:400] Waiting for connection header
09:03:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.428263 (+121626us) server_negotiation.cc:408] Connection header received
09:03:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.573636 (+145373us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
09:03:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.573667 (+ 31us) server_negotiation.cc:462] Received NEGOTIATE request from client
09:03:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.573888 (+ 221us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
09:03:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.574076 (+ 188us) server_negotiation.cc:227] Negotiated authn=SASL
09:03:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.904595 (+330519us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
09:03:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.905756 (+ 1161us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
09:03:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:46.006083 (+100327us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
09:03:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:46.006681 (+ 598us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
09:03:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:46.028675 (+ 21994us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:49784: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:49784): Cannot send after transport endpoint shutdown (error 108)
09:03:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":152}
09:03:46.032 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1039), Traces: [0ms] refreshing cache from master, [73ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.125.60:44533, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.125.61:32775, [228ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.125.62:46583, [1015ms] Sub RPC ConnectToMaster: received response from server master-127.1.125.60:44533: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1015ms] Sub RPC ConnectToMaster: received response from server master-127.1.125.61:32775: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1015ms] Sub RPC ConnectToMaster: received response from server master-127.1.125.62:46583: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.1.125.60:44533,127.1.125.61:32775,127.1.125.62:46583). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.1.125.60:44533,127.1.125.61:32775,127.1.125.62:46583). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
09:03:46.037 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.1.125.60:44533(127.1.125.60:44533)] handling channelInactive
09:03:46.039 [DEBUG - kudu-nio-1] (KuduRpc.java:274) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1045), Traces: [0ms] refreshing cache from master, [73ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.125.60:44533, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.125.61:32775, [228ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.125.62:46583, [1015ms] Sub RPC ConnectToMaster: received response from server master-127.1.125.60:44533: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1015ms] Sub RPC ConnectToMaster: received response from server master-127.1.125.61:32775: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1015ms] Sub RPC ConnectToMaster: received response from server master-127.1.125.62:46583: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) with no deferred attached!
09:03:46.040 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.1.125.61:32775(127.1.125.61:32775)] handling channelInactive
09:03:46.041 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.125.60:44533(127.1.125.60:44533)] disconnecting while in state TERMINATED
09:03:46.042 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.125.61:32775(127.1.125.61:32775)] disconnecting while in state TERMINATED
09:03:46.043 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.125.62:46583(127.1.125.62:46583)] disconnecting while in state TERMINATED
09:03:46.041 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.1.125.62:46583(127.1.125.62:46583)] handling channelInactive
09:03:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:46.046242 2650 negotiation.cc:336] Failed RPC negotiation. Trace:
09:03:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.292114 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:49480 (local address 127.1.125.61:32775)
09:03:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.315675 (+ 23561us) server_negotiation.cc:207] Beginning negotiation
09:03:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.315703 (+ 28us) server_negotiation.cc:400] Waiting for connection header
09:03:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.427676 (+111973us) server_negotiation.cc:408] Connection header received
09:03:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.572582 (+144906us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
09:03:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.572612 (+ 30us) server_negotiation.cc:462] Received NEGOTIATE request from client
09:03:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.572895 (+ 283us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
09:03:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.573126 (+ 231us) server_negotiation.cc:227] Negotiated authn=SASL
09:03:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.906150 (+333024us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
09:03:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:45.907848 (+ 1698us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
09:03:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:46.003634 (+ 95786us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
09:03:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:46.004245 (+ 611us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
09:03:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1116 09:03:46.046186 (+ 41941us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:49480: BlockingRecv error: failed to read from TLS socket (remote: unknown): Cannot send after transport endpoint shutdown (error 108)
09:03:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":22986,"thread_start_us":22715,"threads_started":1}
09:03:46.053 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
09:03:46.063 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
09:03:46.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:46.070389 1524 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu with pid 1869
09:03:46.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:46.101817 1524 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu with pid 2019
09:03:46.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:46.133884 1524 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu with pid 2173
09:03:46.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:46.174525 1524 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu with pid 2523
09:03:46.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:46.207293 1524 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu with pid 2411
09:03:46.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:46.245831 2358 connection.cc:573] client connection to 127.1.125.61:32775 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
09:03:46.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:46.249009 1524 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu with pid 2322
09:03:47.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-16T09:03:47Z chronyd exiting
09:03:47.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:47.677567 1025 debug-util.cc:398] Leaking SignalData structure 0x7b0800098c20 after lost signal to thread 957
09:03:47.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261116 09:03:47.678102 1025 debug-util.cc:398] Leaking SignalData structure 0x7b080009ba00 after lost signal to thread 1028
09:03:48.515 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
09:03:48.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:48.518159 795 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu with pid 1086
09:03:48.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:48.544907 795 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu with pid 1233
09:03:48.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:48.570901 795 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu with pid 1377
09:03:48.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:48.598004 795 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu with pid 807
09:03:48.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:48.627429 795 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu with pid 877
09:03:48.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261116 09:03:48.658195 795 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskQWUX02/build/tsan/bin/kudu with pid 952
09:03:49.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-16T09:03:49Z chronyd exiting
09:03:50.860 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.1.125.60:44533,127.1.125.61:32775,127.1.125.62:46583). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.1.125.60:44533,127.1.125.61:32775,127.1.125.62:46583). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
09:03:52.049 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
09:03:52.049 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.1.125.60:44533,127.1.125.61:32775,127.1.125.62:46583). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.1.125.60:44533,127.1.125.61:32775,127.1.125.62:46583). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more