02:17:51.103 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
02:17:51.115 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskF8zSoX/test-tmp
02:17:51.118 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:17:51.122 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskF8zSoX/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:17:51.385 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
02:17:51.516 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:17:51.518 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:17:51.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:51.521458 12439 mini_oidc.cc:150] Starting JWKS server
02:17:51.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:51.525082 12439 webserver.cc:492] Webserver started at http://127.0.0.1:41763/ using document root <none> and password file <none>
02:17:51.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:51.525663 12439 mini_oidc.cc:186] Starting OIDC Discovery server
02:17:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:51.527508 12439 webserver.cc:492] Webserver started at http://127.0.0.1:42713/ using document root <none> and password file <none>
02:17:51.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-23T02:17:51Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:17:51.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-23T02:17:51Z Disabled control of system clock
02:17:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:51.588959 12439 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:17:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:17:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-0/wal
02:17:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-0/data
02:17:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:17:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:17:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:17:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:17:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:17:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:17:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-0/logs
02:17:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-0/data/info.pb
02:17:51.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:17:51.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:17:51.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:17:51.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:17:51.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:17:51.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:17:51.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:17:51.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:17:51.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:17:51.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:17:51.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:17:51.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:17:51.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.37.254:34821
02:17:51.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.37.254
02:17:51.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:17:51.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.37.212:45885
02:17:51.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:17:51.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:17:51.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:17:51.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:17:51.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.37.254:34821,127.12.37.253:45577,127.12.37.252:34071
02:17:51.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:17:51.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41763/jwks.json with env {}
02:17:51.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:51.889575 12451 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:17:51.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:51.890115 12451 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:17:51.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:51.890645 12451 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:17:51.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:51.922003 12451 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:17:51.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:51.922282 12451 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:17:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:51.922470 12451 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:17:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:51.922649 12451 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:17:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:51.922904 12451 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:17:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:51.923091 12451 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41763/jwks.json
02:17:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:51.966009 12451 master_runner.cc:386] Master server non-default flags:
02:17:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:17:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.37.212:45885
02:17:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:17:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:17:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-0/data
02:17:51.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-0/wal
02:17:51.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:17:51.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.37.254:34821,127.12.37.253:45577,127.12.37.252:34071
02:17:51.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:17:51.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:17:51.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:17:51.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.37.254:34821
02:17:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:17:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:17:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:17:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41763/jwks.json
02:17:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:17:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:17:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-0/data/info.pb
02:17:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.37.254
02:17:51.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:17:51.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:17:51.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:17:51.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:17:51.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:17:51.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:17:51.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-0/logs
02:17:51.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:17:51.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:17:51.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:17:51.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:17:51.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:17:51.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 51f761c37259d8159142e890d4d8106f68bfa520
02:17:51.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:17:51.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 01:43:24 UTC on 5fd53c4cbb9d
02:17:51.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8673
02:17:51.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:17:51.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:51.967368 12451 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:17:51.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:51.968966 12451 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:17:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:51.979899 12457 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:17:53.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:51.979966 12458 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:17:53.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:51.984097 12460 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:17:53.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:53.166607 12459 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:17:53.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.166777 12451 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:17:53.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.171480 12451 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:17:53.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.174048 12451 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:17:53.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.175418 12451 hybrid_clock.cc:648] HybridClock initialized: now 1761185873175375 us; error 63 us; skew 500 ppm
02:17:53.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.176317 12451 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:17:53.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.182448 12451 webserver.cc:492] Webserver started at http://127.12.37.254:38217/ using document root <none> and password file <none>
02:17:53.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.183334 12451 fs_manager.cc:362] Metadata directory not provided
02:17:53.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.183522 12451 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:17:53.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.183918 12451 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:17:53.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.188179 12451 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-0/data/instance:
02:17:53.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9dd853ffb3a54a3ab6e37af9f957a910"
02:17:53.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:17:53 on dist-test-slave-df1k"
02:17:53.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.189317 12451 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-0/wal/instance:
02:17:53.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9dd853ffb3a54a3ab6e37af9f957a910"
02:17:53.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:17:53 on dist-test-slave-df1k"
02:17:53.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.196419 12451 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.000s
02:17:53.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.202107 12467 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:17:53.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.203317 12451 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
02:17:53.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.203588 12451 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-0/data,/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-0/wal
02:17:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9dd853ffb3a54a3ab6e37af9f957a910"
02:17:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:17:53 on dist-test-slave-df1k"
02:17:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.203895 12451 fs_report.cc:389] FS layout report
02:17:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:17:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-0/wal
02:17:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-0/wal
02:17:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-0/data/data
02:17:53.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:17:53.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:17:53.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:17:53.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:17:53.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:17:53.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:17:53.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:17:53.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:17:53.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:17:53.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:17:53.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:17:53.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.263674 12451 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:17:53.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.265126 12451 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:17:53.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.265547 12451 kserver.cc:163] Server-wide thread pool size limit: 3276
02:17:53.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.336370 12451 rpc_server.cc:307] RPC server started. Bound to: 127.12.37.254:34821
02:17:53.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.336529 12518 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.37.254:34821 every 8 connection(s)
02:17:53.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.339205 12451 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-0/data/info.pb
02:17:53.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.344929 12519 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:17:53.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.347514 12439 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu as pid 12451
02:17:53.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.348066 12439 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-0/wal/instance
02:17:53.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.353281 12439 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:17:53.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:17:53.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-1/wal
02:17:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-1/data
02:17:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:17:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:17:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:17:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:17:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:17:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:17:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-1/logs
02:17:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-1/data/info.pb
02:17:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:17:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:17:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:17:53.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:17:53.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:17:53.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:17:53.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:17:53.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:17:53.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:17:53.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:17:53.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:17:53.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:17:53.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.37.253:45577
02:17:53.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.37.253
02:17:53.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:17:53.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.37.212:45885
02:17:53.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:17:53.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:17:53.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:17:53.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:17:53.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.37.254:34821,127.12.37.253:45577,127.12.37.252:34071
02:17:53.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:17:53.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41763/jwks.json with env {}
02:17:53.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.361490 12519 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.37.254" port: 34821 } has no permanent_uuid. Determining permanent_uuid...
02:17:53.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.391623 12519 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } has no permanent_uuid. Determining permanent_uuid...
02:17:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:53.394196 12471 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.37.253:45577: connect: Connection refused (error 111)
02:17:53.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:53.398587 12519 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.37.253:45577: Network error: Client connection negotiation failed: client connection to 127.12.37.253:45577: connect: Connection refused (error 111)
02:17:53.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.448161 12519 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } attempt: 1
02:17:53.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:53.452214 12519 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.37.253:45577: Network error: Client connection negotiation failed: client connection to 127.12.37.253:45577: connect: Connection refused (error 111)
02:17:53.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.520699 12519 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } attempt: 2
02:17:53.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:53.524560 12519 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.37.253:45577: Network error: Client connection negotiation failed: client connection to 127.12.37.253:45577: connect: Connection refused (error 111)
02:17:53.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.616078 12519 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } attempt: 3
02:17:53.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:53.620395 12519 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.37.253:45577: Network error: Client connection negotiation failed: client connection to 127.12.37.253:45577: connect: Connection refused (error 111)
02:17:53.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:53.674222 12521 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:17:53.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:53.674768 12521 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:17:53.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:53.675359 12521 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:17:53.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:53.715616 12521 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:17:53.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:53.715905 12521 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:17:53.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:53.716102 12521 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:17:53.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:53.716287 12521 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:17:53.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:53.716464 12521 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:17:53.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:53.716612 12521 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41763/jwks.json
02:17:53.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.752833 12521 master_runner.cc:386] Master server non-default flags:
02:17:53.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:17:53.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.37.212:45885
02:17:53.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:17:53.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:17:53.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-1/data
02:17:53.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-1/wal
02:17:53.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:17:53.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.37.254:34821,127.12.37.253:45577,127.12.37.252:34071
02:17:53.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:17:53.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:17:53.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:17:53.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.37.253:45577
02:17:53.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:17:53.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:17:53.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:17:53.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41763/jwks.json
02:17:53.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:17:53.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:17:53.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-1/data/info.pb
02:17:53.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.37.253
02:17:53.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:17:53.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:17:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:17:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:17:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:17:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:17:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-1/logs
02:17:53.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:17:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:17:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:17:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:17:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:17:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 51f761c37259d8159142e890d4d8106f68bfa520
02:17:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:17:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 01:43:24 UTC on 5fd53c4cbb9d
02:17:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8673
02:17:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:17:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.754253 12521 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:17:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.755777 12521 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:17:53.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:53.763881 12519 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } attempt: 4
02:17:53.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:53.767635 12530 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:17:53.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:53.769727 12519 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.37.253:45577: Network error: Client connection negotiation failed: client connection to 127.12.37.253:45577: connect: Connection refused (error 111)
02:17:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:54.069224 12519 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } attempt: 5
02:17:54.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:54.077994 12519 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.37.253:45577: Network error: Client connection negotiation failed: client connection to 127.12.37.253:45577: connect: Connection refused (error 111)
02:17:54.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:54.625605 12519 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } attempt: 6
02:17:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:54.656244 12519 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.37.253:45577: Network error: Client connection negotiation failed: client connection to 127.12.37.253:45577: connect: Connection refused (error 111)
02:17:55.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.172009 12529 debug-util.cc:398] Leaking SignalData structure 0x7b0800037520 after lost signal to thread 12521
02:17:55.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.302750 12529 kernel_stack_watchdog.cc:198] Thread 12521 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 402ms:
02:17:55.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:17:55.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:17:55.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:17:55.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:17:55.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.303117 12521 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.536s user 0.544s sys 0.986s
02:17:55.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.303498 12521 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.537s user 0.544s sys 0.986s
02:17:55.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.304010 12532 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1535 milliseconds
02:17:55.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:53.769269 12531 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:17:55.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.305706 12533 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:17:55.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.305629 12521 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:17:55.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.309819 12521 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:17:55.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.312312 12521 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:17:55.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.313653 12521 hybrid_clock.cc:648] HybridClock initialized: now 1761185875313604 us; error 53 us; skew 500 ppm
02:17:55.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.314514 12521 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:17:55.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.320574 12521 webserver.cc:492] Webserver started at http://127.12.37.253:39665/ using document root <none> and password file <none>
02:17:55.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.321633 12521 fs_manager.cc:362] Metadata directory not provided
02:17:55.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.321871 12521 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:17:55.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.322387 12521 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:17:55.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.326719 12521 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-1/data/instance:
02:17:55.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b489c3273e5744508de0cc51ec99572f"
02:17:55.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:17:55 on dist-test-slave-df1k"
02:17:55.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.327930 12521 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-1/wal/instance:
02:17:55.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b489c3273e5744508de0cc51ec99572f"
02:17:55.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:17:55 on dist-test-slave-df1k"
02:17:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.336086 12521 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.005s
02:17:55.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.342303 12542 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:17:55.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.343521 12521 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
02:17:55.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.343919 12521 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-1/data,/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-1/wal
02:17:55.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b489c3273e5744508de0cc51ec99572f"
02:17:55.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:17:55 on dist-test-slave-df1k"
02:17:55.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.344288 12521 fs_report.cc:389] FS layout report
02:17:55.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:17:55.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-1/wal
02:17:55.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-1/wal
02:17:55.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-1/data/data
02:17:55.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:17:55.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:17:55.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:17:55.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:17:55.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:17:55.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:17:55.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:17:55.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:17:55.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:17:55.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:17:55.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:17:55.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.404878 12521 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:17:55.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.406486 12521 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:17:55.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.407006 12521 kserver.cc:163] Server-wide thread pool size limit: 3276
02:17:55.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.480412 12521 rpc_server.cc:307] RPC server started. Bound to: 127.12.37.253:45577
02:17:55.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.480480 12593 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.37.253:45577 every 8 connection(s)
02:17:55.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.483258 12521 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-1/data/info.pb
02:17:55.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.485039 12439 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu as pid 12521
02:17:55.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.485602 12439 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-1/wal/instance
02:17:55.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.488091 12439 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:17:55.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:17:55.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-2/wal
02:17:55.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-2/data
02:17:55.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:17:55.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:17:55.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:17:55.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:17:55.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:17:55.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:17:55.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-2/logs
02:17:55.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-2/data/info.pb
02:17:55.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:17:55.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:17:55.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:17:55.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:17:55.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:17:55.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:17:55.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:17:55.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:17:55.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:17:55.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:17:55.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:17:55.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:17:55.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.37.252:34071
02:17:55.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.37.252
02:17:55.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:17:55.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.37.212:45885
02:17:55.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:17:55.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:17:55.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:17:55.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:17:55.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.37.254:34821,127.12.37.253:45577,127.12.37.252:34071
02:17:55.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:17:55.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41763/jwks.json with env {}
02:17:55.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.489727 12594 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:17:55.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.502763 12594 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.37.254" port: 34821 } has no permanent_uuid. Determining permanent_uuid...
02:17:55.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.523686 12594 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } has no permanent_uuid. Determining permanent_uuid...
02:17:55.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.538237 12594 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } has no permanent_uuid. Determining permanent_uuid...
02:17:55.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.540828 12545 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.37.252:34071: connect: Connection refused (error 111)
02:17:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.545136 12594 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.37.252:34071: Network error: Client connection negotiation failed: client connection to 127.12.37.252:34071: connect: Connection refused (error 111)
02:17:55.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.594755 12594 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } attempt: 1
02:17:55.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.599524 12594 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.37.252:34071: Network error: Client connection negotiation failed: client connection to 127.12.37.252:34071: connect: Connection refused (error 111)
02:17:55.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.668196 12594 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } attempt: 2
02:17:55.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.672181 12594 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.37.252:34071: Network error: Client connection negotiation failed: client connection to 127.12.37.252:34071: connect: Connection refused (error 111)
02:17:55.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.716796 12519 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } attempt: 7
02:17:55.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.727646 12519 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } has no permanent_uuid. Determining permanent_uuid...
02:17:55.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.733069 12519 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.37.252:34071: Network error: Client connection negotiation failed: client connection to 127.12.37.252:34071: connect: Connection refused (error 111)
02:17:55.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.763638 12594 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } attempt: 3
02:17:55.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.768014 12594 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.37.252:34071: Network error: Client connection negotiation failed: client connection to 127.12.37.252:34071: connect: Connection refused (error 111)
02:17:55.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.791572 12519 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } attempt: 1
02:17:55.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.795361 12519 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.37.252:34071: Network error: Client connection negotiation failed: client connection to 127.12.37.252:34071: connect: Connection refused (error 111)
02:17:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.803740 12596 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:17:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.804390 12596 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:17:55.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.805071 12596 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:17:55.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.836990 12596 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:17:55.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.837353 12596 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:17:55.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.837677 12596 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:17:55.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.838035 12596 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:17:55.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.838341 12596 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:17:55.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.838562 12596 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41763/jwks.json
02:17:55.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.875103 12596 master_runner.cc:386] Master server non-default flags:
02:17:55.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:17:55.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.37.212:45885
02:17:55.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:17:55.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:17:55.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-2/data
02:17:55.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-2/wal
02:17:55.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:17:55.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.37.254:34821,127.12.37.253:45577,127.12.37.252:34071
02:17:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:17:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:17:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:17:55.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.37.252:34071
02:17:55.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:17:55.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:17:55.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:17:55.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41763/jwks.json
02:17:55.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:17:55.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:17:55.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-2/data/info.pb
02:17:55.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.37.252
02:17:55.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:17:55.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:17:55.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:17:55.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:17:55.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:17:55.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:17:55.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-2/logs
02:17:55.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:17:55.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:17:55.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:17:55.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:17:55.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:17:55.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 51f761c37259d8159142e890d4d8106f68bfa520
02:17:55.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:17:55.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 01:43:24 UTC on 5fd53c4cbb9d
02:17:55.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8673
02:17:55.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:17:55.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.876739 12596 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:17:55.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.876861 12519 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } attempt: 2
02:17:55.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.878911 12596 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:17:55.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.882153 12519 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.37.252:34071: Network error: Client connection negotiation failed: client connection to 127.12.37.252:34071: connect: Connection refused (error 111)
02:17:55.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.891356 12608 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:17:55.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.911543 12594 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } attempt: 4
02:17:55.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.917347 12594 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.37.252:34071: Network error: Client connection negotiation failed: client connection to 127.12.37.252:34071: connect: Connection refused (error 111)
02:17:55.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:55.967607 12519 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } attempt: 3
02:17:55.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.972537 12519 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.37.252:34071: Network error: Client connection negotiation failed: client connection to 127.12.37.252:34071: connect: Connection refused (error 111)
02:17:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:56.112921 12519 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } attempt: 4
02:17:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:56.117560 12519 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.37.252:34071: Network error: Client connection negotiation failed: client connection to 127.12.37.252:34071: connect: Connection refused (error 111)
02:17:56.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:56.216822 12594 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } attempt: 5
02:17:56.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:56.222196 12594 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.37.252:34071: Network error: Client connection negotiation failed: client connection to 127.12.37.252:34071: connect: Connection refused (error 111)
02:17:56.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:56.400946 12519 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } attempt: 5
02:17:56.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:56.405575 12519 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.37.252:34071: Network error: Client connection negotiation failed: client connection to 127.12.37.252:34071: connect: Connection refused (error 111)
02:17:56.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:56.769598 12594 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } attempt: 6
02:17:56.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:56.782506 12594 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.37.252:34071: Network error: Client connection negotiation failed: client connection to 127.12.37.252:34071: connect: Connection refused (error 111)
02:17:56.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:56.957953 12519 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } attempt: 6
02:17:56.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:56.965304 12519 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.37.252:34071: Network error: Client connection negotiation failed: client connection to 127.12.37.252:34071: connect: Connection refused (error 111)
02:17:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.894672 12609 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:17:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:55.897281 12611 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:17:57.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:57.090757 12610 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:17:57.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.091092 12596 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:17:57.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.094857 12596 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:17:57.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.097364 12596 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:17:57.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.098687 12596 hybrid_clock.cc:648] HybridClock initialized: now 1761185877098662 us; error 45 us; skew 500 ppm
02:17:57.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.099447 12596 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:17:57.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.105436 12596 webserver.cc:492] Webserver started at http://127.12.37.252:38995/ using document root <none> and password file <none>
02:17:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.106303 12596 fs_manager.cc:362] Metadata directory not provided
02:17:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.106488 12596 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:17:57.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.106866 12596 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:17:57.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.111060 12596 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-2/data/instance:
02:17:57.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3112212d94e14e7692691e99b70c60c5"
02:17:57.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:17:57 on dist-test-slave-df1k"
02:17:57.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.112116 12596 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-2/wal/instance:
02:17:57.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3112212d94e14e7692691e99b70c60c5"
02:17:57.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:17:57 on dist-test-slave-df1k"
02:17:57.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.118974 12596 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.004s sys 0.004s
02:17:57.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.124578 12622 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:17:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.125694 12596 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
02:17:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.125975 12596 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-2/data,/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-2/wal
02:17:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3112212d94e14e7692691e99b70c60c5"
02:17:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:17:57 on dist-test-slave-df1k"
02:17:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.126281 12596 fs_report.cc:389] FS layout report
02:17:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:17:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-2/wal
02:17:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-2/wal
02:17:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-2/data/data
02:17:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:17:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:17:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:17:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:17:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:17:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:17:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:17:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:17:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:17:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:17:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:17:57.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.187224 12596 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:17:57.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.188665 12596 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:17:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.189064 12596 kserver.cc:163] Server-wide thread pool size limit: 3276
02:17:57.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.259231 12596 rpc_server.cc:307] RPC server started. Bound to: 127.12.37.252:34071
02:17:57.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.259292 12673 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.37.252:34071 every 8 connection(s)
02:17:57.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.262115 12596 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-2/data/info.pb
02:17:57.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.265430 12439 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu as pid 12596
02:17:57.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.265900 12439 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/master-2/wal/instance
02:17:57.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.269831 12674 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:17:57.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.287793 12674 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.37.254" port: 34821 } has no permanent_uuid. Determining permanent_uuid...
02:17:57.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.312397 12674 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } has no permanent_uuid. Determining permanent_uuid...
02:17:57.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.321820 12674 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } has no permanent_uuid. Determining permanent_uuid...
02:17:57.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.337936 12674 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5: Bootstrap starting.
02:17:57.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.345036 12674 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5: Neither blocks nor log segments found. Creating new log.
02:17:57.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.347764 12674 log.cc:826] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5: Log is configured to *not* fsync() on all Append() calls
02:17:57.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.352983 12674 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5: No bootstrap required, opened a new log
02:17:57.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.374655 12674 raft_consensus.cc:359] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9dd853ffb3a54a3ab6e37af9f957a910" member_type: VOTER last_known_addr { host: "127.12.37.254" port: 34821 } } peers { permanent_uuid: "b489c3273e5744508de0cc51ec99572f" member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } } peers { permanent_uuid: "3112212d94e14e7692691e99b70c60c5" member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } }
02:17:57.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.375312 12674 raft_consensus.cc:385] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:17:57.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.375586 12674 raft_consensus.cc:740] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3112212d94e14e7692691e99b70c60c5, State: Initialized, Role: FOLLOWER
02:17:57.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.376415 12674 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9dd853ffb3a54a3ab6e37af9f957a910" member_type: VOTER last_known_addr { host: "127.12.37.254" port: 34821 } } peers { permanent_uuid: "b489c3273e5744508de0cc51ec99572f" member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } } peers { permanent_uuid: "3112212d94e14e7692691e99b70c60c5" member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } }
02:17:57.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.378682 12683 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9dd853ffb3a54a3ab6e37af9f957a910" member_type: VOTER last_known_addr { host: "127.12.37.254" port: 34821 } } peers { permanent_uuid: "b489c3273e5744508de0cc51ec99572f" member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } } peers { permanent_uuid: "3112212d94e14e7692691e99b70c60c5" member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } } }
02:17:57.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.379276 12683 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [sys.catalog]: This master's current role is: FOLLOWER
02:17:57.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.380221 12674 sys_catalog.cc:565] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [sys.catalog]: configured and running, proceeding with master startup.
02:17:57.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:57.406378 12694 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:17:57.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:57.406730 12694 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:17:57.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.401396 12596 master_runner.cc:186] Error getting master registration for 127.12.37.254:34821: OK, instance_id { permanent_uuid: "9dd853ffb3a54a3ab6e37af9f957a910" instance_seqno: 1761185873317264 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:17:57.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.421375 12596 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:17:57.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.436800 12683 raft_consensus.cc:493] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:17:57.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.437451 12683 raft_consensus.cc:515] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9dd853ffb3a54a3ab6e37af9f957a910" member_type: VOTER last_known_addr { host: "127.12.37.254" port: 34821 } } peers { permanent_uuid: "b489c3273e5744508de0cc51ec99572f" member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } } peers { permanent_uuid: "3112212d94e14e7692691e99b70c60c5" member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } }
02:17:57.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.441499 12683 leader_election.cc:290] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9dd853ffb3a54a3ab6e37af9f957a910 (127.12.37.254:34821), b489c3273e5744508de0cc51ec99572f (127.12.37.253:45577)
02:17:57.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:57.448879 12628 tablet.cc:2378] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:17:57.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.843133 12594 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } attempt: 7
02:17:57.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.865890 12594 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b489c3273e5744508de0cc51ec99572f: Bootstrap starting.
02:17:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.873996 12594 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b489c3273e5744508de0cc51ec99572f: Neither blocks nor log segments found. Creating new log.
02:17:57.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.876600 12594 log.cc:826] T 00000000000000000000000000000000 P b489c3273e5744508de0cc51ec99572f: Log is configured to *not* fsync() on all Append() calls
02:17:57.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.881863 12594 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b489c3273e5744508de0cc51ec99572f: No bootstrap required, opened a new log
02:17:57.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.909201 12594 raft_consensus.cc:359] T 00000000000000000000000000000000 P b489c3273e5744508de0cc51ec99572f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9dd853ffb3a54a3ab6e37af9f957a910" member_type: VOTER last_known_addr { host: "127.12.37.254" port: 34821 } } peers { permanent_uuid: "b489c3273e5744508de0cc51ec99572f" member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } } peers { permanent_uuid: "3112212d94e14e7692691e99b70c60c5" member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } }
02:17:57.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.910465 12594 raft_consensus.cc:385] T 00000000000000000000000000000000 P b489c3273e5744508de0cc51ec99572f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:17:57.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.910777 12594 raft_consensus.cc:740] T 00000000000000000000000000000000 P b489c3273e5744508de0cc51ec99572f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b489c3273e5744508de0cc51ec99572f, State: Initialized, Role: FOLLOWER
02:17:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.911653 12594 consensus_queue.cc:260] T 00000000000000000000000000000000 P b489c3273e5744508de0cc51ec99572f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9dd853ffb3a54a3ab6e37af9f957a910" member_type: VOTER last_known_addr { host: "127.12.37.254" port: 34821 } } peers { permanent_uuid: "b489c3273e5744508de0cc51ec99572f" member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } } peers { permanent_uuid: "3112212d94e14e7692691e99b70c60c5" member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } }
02:17:57.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.916184 12699 sys_catalog.cc:455] T 00000000000000000000000000000000 P b489c3273e5744508de0cc51ec99572f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9dd853ffb3a54a3ab6e37af9f957a910" member_type: VOTER last_known_addr { host: "127.12.37.254" port: 34821 } } peers { permanent_uuid: "b489c3273e5744508de0cc51ec99572f" member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } } peers { permanent_uuid: "3112212d94e14e7692691e99b70c60c5" member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } } }
02:17:57.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.917596 12594 sys_catalog.cc:565] T 00000000000000000000000000000000 P b489c3273e5744508de0cc51ec99572f [sys.catalog]: configured and running, proceeding with master startup.
02:17:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.917745 12569 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3112212d94e14e7692691e99b70c60c5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b489c3273e5744508de0cc51ec99572f" is_pre_election: true
02:17:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.918932 12569 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b489c3273e5744508de0cc51ec99572f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3112212d94e14e7692691e99b70c60c5 in term 0.
02:17:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.920332 12626 leader_election.cc:304] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3112212d94e14e7692691e99b70c60c5, b489c3273e5744508de0cc51ec99572f; no voters:
02:17:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.921217 12683 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [term 0 FOLLOWER]: Leader pre-election won for term 1
02:17:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.921557 12683 raft_consensus.cc:493] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:17:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.921862 12683 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [term 0 FOLLOWER]: Advancing to term 1
02:17:57.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.935555 12683 raft_consensus.cc:515] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9dd853ffb3a54a3ab6e37af9f957a910" member_type: VOTER last_known_addr { host: "127.12.37.254" port: 34821 } } peers { permanent_uuid: "b489c3273e5744508de0cc51ec99572f" member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } } peers { permanent_uuid: "3112212d94e14e7692691e99b70c60c5" member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } }
02:17:57.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.918356 12699 sys_catalog.cc:458] T 00000000000000000000000000000000 P b489c3273e5744508de0cc51ec99572f [sys.catalog]: This master's current role is: FOLLOWER
02:17:57.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.939389 12683 leader_election.cc:290] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [CANDIDATE]: Term 1 election: Requested vote from peers 9dd853ffb3a54a3ab6e37af9f957a910 (127.12.37.254:34821), b489c3273e5744508de0cc51ec99572f (127.12.37.253:45577)
02:17:57.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.940317 12569 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3112212d94e14e7692691e99b70c60c5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b489c3273e5744508de0cc51ec99572f"
02:17:57.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.940765 12569 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b489c3273e5744508de0cc51ec99572f [term 0 FOLLOWER]: Advancing to term 1
02:17:57.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.950088 12569 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b489c3273e5744508de0cc51ec99572f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3112212d94e14e7692691e99b70c60c5 in term 1.
02:17:57.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.951258 12626 leader_election.cc:304] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3112212d94e14e7692691e99b70c60c5, b489c3273e5744508de0cc51ec99572f; no voters:
02:17:57.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.952137 12683 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [term 1 FOLLOWER]: Leader election won for term 1
02:17:57.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.959170 12683 raft_consensus.cc:697] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [term 1 LEADER]: Becoming Leader. State: Replica: 3112212d94e14e7692691e99b70c60c5, State: Running, Role: LEADER
02:17:57.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.960580 12683 consensus_queue.cc:237] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9dd853ffb3a54a3ab6e37af9f957a910" member_type: VOTER last_known_addr { host: "127.12.37.254" port: 34821 } } peers { permanent_uuid: "b489c3273e5744508de0cc51ec99572f" member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } } peers { permanent_uuid: "3112212d94e14e7692691e99b70c60c5" member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } }
02:17:57.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.976905 12710 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3112212d94e14e7692691e99b70c60c5. Latest consensus state: current_term: 1 leader_uuid: "3112212d94e14e7692691e99b70c60c5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9dd853ffb3a54a3ab6e37af9f957a910" member_type: VOTER last_known_addr { host: "127.12.37.254" port: 34821 } } peers { permanent_uuid: "b489c3273e5744508de0cc51ec99572f" member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } } peers { permanent_uuid: "3112212d94e14e7692691e99b70c60c5" member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } } }
02:17:57.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.978371 12710 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [sys.catalog]: This master's current role is: LEADER
02:17:57.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.990594 12714 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
02:17:57.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:57.995519 12711 catalog_manager.cc:1568] T 00000000000000000000000000000000 P b489c3273e5744508de0cc51ec99572f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:17:57.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:57.996201 12711 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:17:57.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.997867 12714 catalog_manager.cc:1494] Initializing Kudu cluster ID...
02:17:58.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.998658 12519 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } attempt: 7
02:17:58.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:57.966159 12521 master_runner.cc:186] Error getting master registration for 127.12.37.254:34821: OK, instance_id { permanent_uuid: "9dd853ffb3a54a3ab6e37af9f957a910" instance_seqno: 1761185873317264 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:17:58.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.008836 12521 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:17:58.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.036091 12569 raft_consensus.cc:1275] T 00000000000000000000000000000000 P b489c3273e5744508de0cc51ec99572f [term 1 FOLLOWER]: Refusing update from remote peer 3112212d94e14e7692691e99b70c60c5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:17:58.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.038245 12710 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b489c3273e5744508de0cc51ec99572f" member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:17:58.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.055055 12519 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9dd853ffb3a54a3ab6e37af9f957a910: Bootstrap starting.
02:17:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.067219 12519 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9dd853ffb3a54a3ab6e37af9f957a910: Neither blocks nor log segments found. Creating new log.
02:17:58.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.073902 12699 sys_catalog.cc:455] T 00000000000000000000000000000000 P b489c3273e5744508de0cc51ec99572f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3112212d94e14e7692691e99b70c60c5. Latest consensus state: current_term: 1 leader_uuid: "3112212d94e14e7692691e99b70c60c5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9dd853ffb3a54a3ab6e37af9f957a910" member_type: VOTER last_known_addr { host: "127.12.37.254" port: 34821 } } peers { permanent_uuid: "b489c3273e5744508de0cc51ec99572f" member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } } peers { permanent_uuid: "3112212d94e14e7692691e99b70c60c5" member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } } }
02:17:58.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.074616 12699 sys_catalog.cc:458] T 00000000000000000000000000000000 P b489c3273e5744508de0cc51ec99572f [sys.catalog]: This master's current role is: FOLLOWER
02:17:58.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.080328 12519 log.cc:826] T 00000000000000000000000000000000 P 9dd853ffb3a54a3ab6e37af9f957a910: Log is configured to *not* fsync() on all Append() calls
02:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.085042 12716 mvcc.cc:204] Tried to move back new op lower bound from 7213817356383920128 to 7213817356137730048. Current Snapshot: MvccSnapshot[applied={T|T < 7213817356383920128}]
02:17:58.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.093827 12683 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "3112212d94e14e7692691e99b70c60c5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9dd853ffb3a54a3ab6e37af9f957a910" member_type: VOTER last_known_addr { host: "127.12.37.254" port: 34821 } } peers { permanent_uuid: "b489c3273e5744508de0cc51ec99572f" member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } } peers { permanent_uuid: "3112212d94e14e7692691e99b70c60c5" member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } } }
02:17:58.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.094549 12683 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [sys.catalog]: This master's current role is: LEADER
02:17:58.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.098887 12699 sys_catalog.cc:455] T 00000000000000000000000000000000 P b489c3273e5744508de0cc51ec99572f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "3112212d94e14e7692691e99b70c60c5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9dd853ffb3a54a3ab6e37af9f957a910" member_type: VOTER last_known_addr { host: "127.12.37.254" port: 34821 } } peers { permanent_uuid: "b489c3273e5744508de0cc51ec99572f" member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } } peers { permanent_uuid: "3112212d94e14e7692691e99b70c60c5" member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } } }
02:17:58.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.099597 12699 sys_catalog.cc:458] T 00000000000000000000000000000000 P b489c3273e5744508de0cc51ec99572f [sys.catalog]: This master's current role is: FOLLOWER
02:17:58.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.097306 12717 mvcc.cc:204] Tried to move back new op lower bound from 7213817356383920128 to 7213817356137730048. Current Snapshot: MvccSnapshot[applied={T|T < 7213817356383920128}]
02:17:58.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.105505 12714 catalog_manager.cc:1357] Generated new cluster ID: bbe359257a394a93934a10c4429c35eb
02:17:58.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.105813 12714 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
02:17:58.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.120500 12519 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9dd853ffb3a54a3ab6e37af9f957a910: No bootstrap required, opened a new log
02:17:58.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.161906 12519 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9dd853ffb3a54a3ab6e37af9f957a910 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9dd853ffb3a54a3ab6e37af9f957a910" member_type: VOTER last_known_addr { host: "127.12.37.254" port: 34821 } } peers { permanent_uuid: "b489c3273e5744508de0cc51ec99572f" member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } } peers { permanent_uuid: "3112212d94e14e7692691e99b70c60c5" member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } }
02:17:58.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.163190 12519 raft_consensus.cc:385] T 00000000000000000000000000000000 P 9dd853ffb3a54a3ab6e37af9f957a910 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:17:58.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.163715 12519 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9dd853ffb3a54a3ab6e37af9f957a910 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9dd853ffb3a54a3ab6e37af9f957a910, State: Initialized, Role: FOLLOWER
02:17:58.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.164880 12519 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9dd853ffb3a54a3ab6e37af9f957a910 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9dd853ffb3a54a3ab6e37af9f957a910" member_type: VOTER last_known_addr { host: "127.12.37.254" port: 34821 } } peers { permanent_uuid: "b489c3273e5744508de0cc51ec99572f" member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } } peers { permanent_uuid: "3112212d94e14e7692691e99b70c60c5" member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } }
02:17:58.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.167560 12714 catalog_manager.cc:1380] Generated new certificate authority record
02:17:58.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.173380 12714 catalog_manager.cc:1514] Loading token signing keys...
02:17:58.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.176934 12519 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9dd853ffb3a54a3ab6e37af9f957a910 [sys.catalog]: configured and running, proceeding with master startup.
02:17:58.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.177130 12494 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3112212d94e14e7692691e99b70c60c5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9dd853ffb3a54a3ab6e37af9f957a910" is_pre_election: true
02:17:58.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.178053 12494 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9dd853ffb3a54a3ab6e37af9f957a910 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3112212d94e14e7692691e99b70c60c5 in term 0.
02:17:58.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.179044 12493 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3112212d94e14e7692691e99b70c60c5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9dd853ffb3a54a3ab6e37af9f957a910"
02:17:58.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.179634 12493 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9dd853ffb3a54a3ab6e37af9f957a910 [term 0 FOLLOWER]: Advancing to term 1
02:17:58.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.173502 12722 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9dd853ffb3a54a3ab6e37af9f957a910 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9dd853ffb3a54a3ab6e37af9f957a910" member_type: VOTER last_known_addr { host: "127.12.37.254" port: 34821 } } peers { permanent_uuid: "b489c3273e5744508de0cc51ec99572f" member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } } peers { permanent_uuid: "3112212d94e14e7692691e99b70c60c5" member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } } }
02:17:58.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.181147 12722 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9dd853ffb3a54a3ab6e37af9f957a910 [sys.catalog]: This master's current role is: FOLLOWER
02:17:58.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.190511 12493 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9dd853ffb3a54a3ab6e37af9f957a910 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3112212d94e14e7692691e99b70c60c5 in term 1.
02:17:58.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.192063 12492 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 9dd853ffb3a54a3ab6e37af9f957a910 [term 1 FOLLOWER]: Refusing update from remote peer 3112212d94e14e7692691e99b70c60c5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:17:58.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.200819 12683 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9dd853ffb3a54a3ab6e37af9f957a910" member_type: VOTER last_known_addr { host: "127.12.37.254" port: 34821 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
02:17:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.265571 12714 catalog_manager.cc:6022] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5: Generated new TSK 0
02:17:58.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.266997 12714 catalog_manager.cc:1524] Initializing in-progress tserver states...
02:17:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.286880 12722 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9dd853ffb3a54a3ab6e37af9f957a910 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3112212d94e14e7692691e99b70c60c5. Latest consensus state: current_term: 1 leader_uuid: "3112212d94e14e7692691e99b70c60c5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9dd853ffb3a54a3ab6e37af9f957a910" member_type: VOTER last_known_addr { host: "127.12.37.254" port: 34821 } } peers { permanent_uuid: "b489c3273e5744508de0cc51ec99572f" member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } } peers { permanent_uuid: "3112212d94e14e7692691e99b70c60c5" member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } } }
02:17:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.287014 12735 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9dd853ffb3a54a3ab6e37af9f957a910 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "3112212d94e14e7692691e99b70c60c5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9dd853ffb3a54a3ab6e37af9f957a910" member_type: VOTER last_known_addr { host: "127.12.37.254" port: 34821 } } peers { permanent_uuid: "b489c3273e5744508de0cc51ec99572f" member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } } peers { permanent_uuid: "3112212d94e14e7692691e99b70c60c5" member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } } }
02:17:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.287606 12722 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9dd853ffb3a54a3ab6e37af9f957a910 [sys.catalog]: This master's current role is: FOLLOWER
02:17:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.287828 12735 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9dd853ffb3a54a3ab6e37af9f957a910 [sys.catalog]: This master's current role is: FOLLOWER
02:17:58.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.291462 12710 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "3112212d94e14e7692691e99b70c60c5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9dd853ffb3a54a3ab6e37af9f957a910" member_type: VOTER last_known_addr { host: "127.12.37.254" port: 34821 } } peers { permanent_uuid: "b489c3273e5744508de0cc51ec99572f" member_type: VOTER last_known_addr { host: "127.12.37.253" port: 45577 } } peers { permanent_uuid: "3112212d94e14e7692691e99b70c60c5" member_type: VOTER last_known_addr { host: "127.12.37.252" port: 34071 } } }
02:17:58.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.292161 12710 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3112212d94e14e7692691e99b70c60c5 [sys.catalog]: This master's current role is: LEADER
02:17:58.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.318133 12439 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:17:58.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:17:58.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-0/wal
02:17:58.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-0/data
02:17:58.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:17:58.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:17:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:17:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:17:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:17:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:17:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-0/logs
02:17:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-0/data/info.pb
02:17:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:17:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:17:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:17:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:17:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:17:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:17:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:17:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:17:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:17:58.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:17:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.37.193:0
02:17:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.37.193
02:17:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.37.193
02:17:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:17:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.37.254:34821,127.12.37.253:45577,127.12.37.252:34071
02:17:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.37.212:45885
02:17:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:17:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:17:58.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:17:58.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.339819 12738 catalog_manager.cc:1269] Loaded cluster ID: bbe359257a394a93934a10c4429c35eb
02:17:58.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.349609 12738 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 9dd853ffb3a54a3ab6e37af9f957a910: loading cluster ID for follower catalog manager: success
02:17:58.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.359061 12738 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 9dd853ffb3a54a3ab6e37af9f957a910: acquiring CA information for follower catalog manager: success
02:17:58.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:58.371747 12738 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 9dd853ffb3a54a3ab6e37af9f957a910: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:17:58.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:58.923202 12741 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:17:58.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:58.924217 12741 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:17:58.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:58.926088 12741 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:17:58.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:58.987514 12741 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:17:58.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:58.989315 12741 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.37.193
02:17:59.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:59.002019 12711 catalog_manager.cc:1269] Loaded cluster ID: bbe359257a394a93934a10c4429c35eb
02:17:59.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:59.002324 12711 catalog_manager.cc:1562] T 00000000000000000000000000000000 P b489c3273e5744508de0cc51ec99572f: loading cluster ID for follower catalog manager: success
02:17:59.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:59.018599 12711 catalog_manager.cc:1584] T 00000000000000000000000000000000 P b489c3273e5744508de0cc51ec99572f: acquiring CA information for follower catalog manager: success
02:17:59.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:59.024313 12711 catalog_manager.cc:1612] T 00000000000000000000000000000000 P b489c3273e5744508de0cc51ec99572f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:17:59.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:59.059257 12741 tablet_server_runner.cc:78] Tablet server non-default flags:
02:17:59.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:17:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.37.212:45885
02:17:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:17:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:17:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-0/data
02:17:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-0/wal
02:17:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:17:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:17:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.37.193:0
02:17:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:17:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:17:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:17:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-0/data/info.pb
02:17:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.37.193
02:17:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:17:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.37.254:34821,127.12.37.253:45577,127.12.37.252:34071
02:17:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:17:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:17:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:17:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:17:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:17:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.37.193
02:17:59.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-0/logs
02:17:59.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:17:59.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:17:59.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:17:59.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:17:59.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:17:59.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 51f761c37259d8159142e890d4d8106f68bfa520
02:17:59.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:17:59.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 01:43:24 UTC on 5fd53c4cbb9d
02:17:59.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8673
02:17:59.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:17:59.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:59.068130 12741 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:17:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:17:59.070938 12741 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:17:59.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:59.102167 12747 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:01.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:00.493841 12746 debug-util.cc:398] Leaking SignalData structure 0x7b0800037500 after lost signal to thread 12741
02:18:01.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:01.315623 12746 kernel_stack_watchdog.cc:198] Thread 12741 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 385ms:
02:18:01.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:18:01.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:18:01.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:18:01.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:18:01.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:17:59.107612 12748 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:01.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:01.317979 12741 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.211s user 0.434s sys 0.777s
02:18:01.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:01.318101 12751 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:01.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:01.318336 12741 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.212s user 0.434s sys 0.777s
02:18:01.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:01.321835 12749 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2216 milliseconds
02:18:01.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.321864 12741 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:18:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.323666 12741 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:18:01.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.326596 12741 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:18:01.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.328193 12741 hybrid_clock.cc:648] HybridClock initialized: now 1761185881328156 us; error 44 us; skew 500 ppm
02:18:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.329478 12741 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:18:01.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.368983 12741 webserver.cc:492] Webserver started at http://127.12.37.193:46355/ using document root <none> and password file <none>
02:18:01.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.370858 12741 fs_manager.cc:362] Metadata directory not provided
02:18:01.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.371150 12741 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:18:01.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.371695 12741 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:18:01.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.378072 12741 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-0/data/instance:
02:18:01.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "10e7817de8f94e75834d879dc3f7331a"
02:18:01.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:01 on dist-test-slave-df1k"
02:18:01.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.379804 12741 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-0/wal/instance:
02:18:01.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "10e7817de8f94e75834d879dc3f7331a"
02:18:01.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:01 on dist-test-slave-df1k"
02:18:01.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.390206 12741 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.009s sys 0.001s
02:18:01.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.399683 12759 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:18:01.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.401515 12741 fs_manager.cc:730] Time spent opening block manager: real 0.007s user 0.005s sys 0.000s
02:18:01.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.401934 12741 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-0/data,/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-0/wal
02:18:01.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "10e7817de8f94e75834d879dc3f7331a"
02:18:01.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:01 on dist-test-slave-df1k"
02:18:01.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.402380 12741 fs_report.cc:389] FS layout report
02:18:01.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:18:01.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-0/wal
02:18:01.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-0/wal
02:18:01.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-0/data/data
02:18:01.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:18:01.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:18:01.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:18:01.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:18:01.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:18:01.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:18:01.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:18:01.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:18:01.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:18:01.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:18:01.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:18:01.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.479285 12741 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:18:01.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.481269 12741 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:18:01.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.481809 12741 kserver.cc:163] Server-wide thread pool size limit: 3276
02:18:01.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.498054 12741 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:18:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.510896 12741 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:18:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.511211 12741 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:18:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.511492 12741 ts_tablet_manager.cc:616] Registered 0 tablets
02:18:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.511696 12741 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:18:01.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.775399 12741 rpc_server.cc:307] RPC server started. Bound to: 127.12.37.193:39409
02:18:01.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.775528 12872 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.37.193:39409 every 8 connection(s)
02:18:01.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.778936 12741 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-0/data/info.pb
02:18:01.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.788997 12439 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu as pid 12741
02:18:01.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.789822 12439 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-0/wal/instance
02:18:01.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.804661 12439 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:18:01.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:18:01.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-1/wal
02:18:01.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-1/data
02:18:01.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:18:01.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:18:01.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:18:01.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:18:01.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:18:01.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:18:01.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-1/logs
02:18:01.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-1/data/info.pb
02:18:01.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:18:01.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:18:01.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:18:01.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:18:01.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:18:01.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:18:01.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:18:01.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:18:01.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:18:01.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:18:01.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.37.194:0
02:18:01.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.37.194
02:18:01.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.37.194
02:18:01.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:18:01.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.37.254:34821,127.12.37.253:45577,127.12.37.252:34071
02:18:01.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.37.212:45885
02:18:01.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:18:01.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:18:01.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:18:01.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.830660 12873 heartbeater.cc:344] Connected to a master server at 127.12.37.253:45577
02:18:01.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.831164 12873 heartbeater.cc:461] Registering TS with master...
02:18:01.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.832403 12873 heartbeater.cc:507] Master 127.12.37.253:45577 requested a full tablet report, sending...
02:18:01.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.836660 12559 ts_manager.cc:194] Registered new tserver with Master: 10e7817de8f94e75834d879dc3f7331a (127.12.37.193:39409)
02:18:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.872355 12877 heartbeater.cc:344] Connected to a master server at 127.12.37.254:34821
02:18:01.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.872752 12877 heartbeater.cc:461] Registering TS with master...
02:18:01.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.873641 12877 heartbeater.cc:507] Master 127.12.37.254:34821 requested a full tablet report, sending...
02:18:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.877434 12484 ts_manager.cc:194] Registered new tserver with Master: 10e7817de8f94e75834d879dc3f7331a (127.12.37.193:39409)
02:18:01.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.883396 12874 heartbeater.cc:344] Connected to a master server at 127.12.37.252:34071
02:18:01.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.883826 12874 heartbeater.cc:461] Registering TS with master...
02:18:01.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.884716 12874 heartbeater.cc:507] Master 127.12.37.252:34071 requested a full tablet report, sending...
02:18:01.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.889670 12639 ts_manager.cc:194] Registered new tserver with Master: 10e7817de8f94e75834d879dc3f7331a (127.12.37.193:39409)
02:18:01.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:01.892894 12639 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.37.193:58539
02:18:02.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:02.182592 12879 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:18:02.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:02.183107 12879 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:18:02.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:02.183527 12879 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:18:02.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:02.216089 12879 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:18:02.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:02.216902 12879 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.37.194
02:18:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:02.253274 12879 tablet_server_runner.cc:78] Tablet server non-default flags:
02:18:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:18:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.37.212:45885
02:18:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:18:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:18:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-1/data
02:18:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-1/wal
02:18:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:18:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:18:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.37.194:0
02:18:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:18:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:18:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:18:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-1/data/info.pb
02:18:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.37.194
02:18:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:18:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.37.254:34821,127.12.37.253:45577,127.12.37.252:34071
02:18:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:18:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:18:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:18:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:18:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:18:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.37.194
02:18:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-1/logs
02:18:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:18:02.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:18:02.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:18:02.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:18:02.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:18:02.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 51f761c37259d8159142e890d4d8106f68bfa520
02:18:02.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:18:02.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 01:43:24 UTC on 5fd53c4cbb9d
02:18:02.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8673
02:18:02.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:18:02.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:02.254683 12879 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:18:02.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:02.256433 12879 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:18:02.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:02.272746 12890 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:02.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:02.898087 12874 heartbeater.cc:499] Master 127.12.37.252:34071 was elected leader, sending a full tablet report...
02:18:03.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:03.678695 12889 debug-util.cc:398] Leaking SignalData structure 0x7b0800037500 after lost signal to thread 12879
02:18:03.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:03.817869 12879 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.544s user 0.526s sys 0.998s
02:18:03.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:03.818261 12879 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.544s user 0.526s sys 0.998s
02:18:03.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:02.274828 12891 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:03.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:03.820330 12893 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:03.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:03.823383 12892 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1545 milliseconds
02:18:03.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.823460 12879 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:18:03.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.824599 12879 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:18:03.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.826707 12879 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:18:03.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.828079 12879 hybrid_clock.cc:648] HybridClock initialized: now 1761185883828037 us; error 48 us; skew 500 ppm
02:18:03.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.829066 12879 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:18:03.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.835320 12879 webserver.cc:492] Webserver started at http://127.12.37.194:34063/ using document root <none> and password file <none>
02:18:03.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.836297 12879 fs_manager.cc:362] Metadata directory not provided
02:18:03.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.836535 12879 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:18:03.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.837037 12879 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:18:03.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.841326 12879 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-1/data/instance:
02:18:03.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1340c5a639a341b285e31fb36919a73f"
02:18:03.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:03 on dist-test-slave-df1k"
02:18:03.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.842483 12879 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-1/wal/instance:
02:18:03.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1340c5a639a341b285e31fb36919a73f"
02:18:03.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:03 on dist-test-slave-df1k"
02:18:03.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.850049 12879 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.002s
02:18:03.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.855984 12901 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:18:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.857107 12879 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.001s
02:18:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.857509 12879 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-1/data,/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-1/wal
02:18:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1340c5a639a341b285e31fb36919a73f"
02:18:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:03 on dist-test-slave-df1k"
02:18:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.857851 12879 fs_report.cc:389] FS layout report
02:18:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:18:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-1/wal
02:18:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-1/wal
02:18:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-1/data/data
02:18:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:18:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:18:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:18:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:18:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:18:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:18:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:18:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:18:03.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:18:03.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:18:03.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:18:03.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.906705 12879 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:18:03.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.908262 12879 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:18:03.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.908752 12879 kserver.cc:163] Server-wide thread pool size limit: 3276
02:18:03.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.911300 12879 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:18:03.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.915671 12879 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:18:03.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.915887 12879 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:18:03.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.916215 12879 ts_tablet_manager.cc:616] Registered 0 tablets
02:18:03.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:03.916390 12879 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:18:04.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:04.072417 12879 rpc_server.cc:307] RPC server started. Bound to: 127.12.37.194:35425
02:18:04.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:04.072497 13013 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.37.194:35425 every 8 connection(s)
02:18:04.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:04.075078 12879 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-1/data/info.pb
02:18:04.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:04.076493 12439 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu as pid 12879
02:18:04.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:04.076892 12439 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-1/wal/instance
02:18:04.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:04.088457 12439 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:18:04.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:18:04.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-2/wal
02:18:04.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-2/data
02:18:04.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:18:04.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:18:04.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:18:04.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:18:04.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:18:04.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:18:04.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-2/logs
02:18:04.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-2/data/info.pb
02:18:04.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:18:04.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:18:04.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:18:04.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:18:04.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:18:04.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:18:04.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:18:04.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:18:04.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:18:04.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:18:04.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.37.195:0
02:18:04.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.37.195
02:18:04.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.37.195
02:18:04.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:18:04.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.37.254:34821,127.12.37.253:45577,127.12.37.252:34071
02:18:04.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.37.212:45885
02:18:04.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:18:04.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:18:04.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:18:04.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:04.137938 13014 heartbeater.cc:344] Connected to a master server at 127.12.37.253:45577
02:18:04.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:04.138511 13014 heartbeater.cc:461] Registering TS with master...
02:18:04.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:04.140026 13014 heartbeater.cc:507] Master 127.12.37.253:45577 requested a full tablet report, sending...
02:18:04.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:04.140393 13016 heartbeater.cc:344] Connected to a master server at 127.12.37.254:34821
02:18:04.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:04.140767 13016 heartbeater.cc:461] Registering TS with master...
02:18:04.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:04.141626 13016 heartbeater.cc:507] Master 127.12.37.254:34821 requested a full tablet report, sending...
02:18:04.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:04.144172 12559 ts_manager.cc:194] Registered new tserver with Master: 1340c5a639a341b285e31fb36919a73f (127.12.37.194:35425)
02:18:04.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:04.145359 13015 heartbeater.cc:344] Connected to a master server at 127.12.37.252:34071
02:18:04.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:04.145718 13015 heartbeater.cc:461] Registering TS with master...
02:18:04.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:04.145939 12484 ts_manager.cc:194] Registered new tserver with Master: 1340c5a639a341b285e31fb36919a73f (127.12.37.194:35425)
02:18:04.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:04.146552 13015 heartbeater.cc:507] Master 127.12.37.252:34071 requested a full tablet report, sending...
02:18:04.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:04.149909 12639 ts_manager.cc:194] Registered new tserver with Master: 1340c5a639a341b285e31fb36919a73f (127.12.37.194:35425)
02:18:04.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:04.151544 12639 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.37.194:48633
02:18:04.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:04.450955 13021 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:18:04.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:04.451406 13021 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:18:04.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:04.451834 13021 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:18:04.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:04.485200 13021 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:18:04.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:04.486135 13021 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.37.195
02:18:04.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:04.523582 13021 tablet_server_runner.cc:78] Tablet server non-default flags:
02:18:04.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:18:04.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.37.212:45885
02:18:04.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:18:04.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:18:04.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-2/data
02:18:04.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-2/wal
02:18:04.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:18:04.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:18:04.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.37.195:0
02:18:04.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:18:04.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:18:04.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:18:04.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-2/data/info.pb
02:18:04.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.37.195
02:18:04.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:18:04.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.37.254:34821,127.12.37.253:45577,127.12.37.252:34071
02:18:04.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:18:04.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:18:04.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:18:04.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:18:04.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:18:04.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.37.195
02:18:04.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-2/logs
02:18:04.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:18:04.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:18:04.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:18:04.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:18:04.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:18:04.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 51f761c37259d8159142e890d4d8106f68bfa520
02:18:04.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:18:04.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 01:43:24 UTC on 5fd53c4cbb9d
02:18:04.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8673
02:18:04.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:18:04.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:04.524880 13021 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:18:04.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:04.526400 13021 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:18:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:04.539408 13030 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:05.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:05.154918 13015 heartbeater.cc:499] Master 127.12.37.252:34071 was elected leader, sending a full tablet report...
02:18:06.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:04.540627 13031 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:06.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:06.304441 13021 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.764s user 0.646s sys 1.112s
02:18:06.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:06.305089 13021 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.765s user 0.646s sys 1.112s
02:18:06.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:05.942976 13029 debug-util.cc:398] Leaking SignalData structure 0x7b0800037500 after lost signal to thread 13021
02:18:06.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:06.307121 13033 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:06.311036 13032 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1769 milliseconds
02:18:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.311071 13021 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:18:06.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.312297 13021 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:18:06.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.314579 13021 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:18:06.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.315932 13021 hybrid_clock.cc:648] HybridClock initialized: now 1761185886315889 us; error 55 us; skew 500 ppm
02:18:06.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.316847 13021 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:18:06.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.322932 13021 webserver.cc:492] Webserver started at http://127.12.37.195:38965/ using document root <none> and password file <none>
02:18:06.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.323872 13021 fs_manager.cc:362] Metadata directory not provided
02:18:06.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.324088 13021 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:18:06.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.324554 13021 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:18:06.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.328804 13021 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-2/data/instance:
02:18:06.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d8439adb2cc941988298631987faed75"
02:18:06.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:06 on dist-test-slave-df1k"
02:18:06.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.329949 13021 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-2/wal/instance:
02:18:06.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d8439adb2cc941988298631987faed75"
02:18:06.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:06 on dist-test-slave-df1k"
02:18:06.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.336912 13021 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.001s
02:18:06.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.342995 13044 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:18:06.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.344275 13021 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
02:18:06.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.344656 13021 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-2/data,/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-2/wal
02:18:06.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d8439adb2cc941988298631987faed75"
02:18:06.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:06 on dist-test-slave-df1k"
02:18:06.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.345085 13021 fs_report.cc:389] FS layout report
02:18:06.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:18:06.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-2/wal
02:18:06.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-2/wal
02:18:06.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-2/data/data
02:18:06.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:18:06.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:18:06.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:18:06.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:18:06.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:18:06.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:18:06.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:18:06.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:18:06.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:18:06.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:18:06.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:18:06.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.412168 13021 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:18:06.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.413625 13021 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:18:06.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.414083 13021 kserver.cc:163] Server-wide thread pool size limit: 3276
02:18:06.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.416517 13021 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:18:06.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.420847 13021 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:18:06.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.421036 13021 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:18:06.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.421228 13021 ts_tablet_manager.cc:616] Registered 0 tablets
02:18:06.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.421381 13021 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.001s
02:18:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.594985 13021 rpc_server.cc:307] RPC server started. Bound to: 127.12.37.195:35043
02:18:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.595096 13156 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.37.195:35043 every 8 connection(s)
02:18:06.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.597702 13021 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-2/data/info.pb
02:18:06.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.602136 12439 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu as pid 13021
02:18:06.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.602583 12439 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster6645735047260563036/ts-2/wal/instance
02:18:06.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.642555 13159 heartbeater.cc:344] Connected to a master server at 127.12.37.254:34821
02:18:06.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.642818 13158 heartbeater.cc:344] Connected to a master server at 127.12.37.252:34071
02:18:06.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.643106 13157 heartbeater.cc:344] Connected to a master server at 127.12.37.253:45577
02:18:06.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.643204 13158 heartbeater.cc:461] Registering TS with master...
02:18:06.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.643460 13157 heartbeater.cc:461] Registering TS with master...
02:18:06.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.643126 13159 heartbeater.cc:461] Registering TS with master...
02:18:06.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.645303 13157 heartbeater.cc:507] Master 127.12.37.253:45577 requested a full tablet report, sending...
02:18:06.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.645339 13159 heartbeater.cc:507] Master 127.12.37.254:34821 requested a full tablet report, sending...
02:18:06.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.645327 13158 heartbeater.cc:507] Master 127.12.37.252:34071 requested a full tablet report, sending...
02:18:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.648424 12639 ts_manager.cc:194] Registered new tserver with Master: d8439adb2cc941988298631987faed75 (127.12.37.195:35043)
02:18:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.648617 12559 ts_manager.cc:194] Registered new tserver with Master: d8439adb2cc941988298631987faed75 (127.12.37.195:35043)
02:18:06.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.648681 12483 ts_manager.cc:194] Registered new tserver with Master: d8439adb2cc941988298631987faed75 (127.12.37.195:35043)
02:18:06.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.650629 12639 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.12.37.195:49277
02:18:06.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:06.669528 12439 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
02:18:06.681 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:18:06.683 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
02:18:06.698 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.12.37.254"
port: 34821
}
bound_http_address {
host: "127.12.37.254"
port: 38217
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.12.37.253"
port: 45577
}
bound_http_address {
host: "127.12.37.253"
port: 39665
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.12.37.252"
port: 34071
}
bound_http_address {
host: "127.12.37.252"
port: 38995
}
}
}
02:18:06.703 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:18:06.706 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.12.37.193"
port: 39409
}
bound_http_address {
host: "127.12.37.193"
port: 46355
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.12.37.194"
port: 35425
}
bound_http_address {
host: "127.12.37.194"
port: 34063
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.12.37.195"
port: 35043
}
bound_http_address {
host: "127.12.37.195"
port: 38965
}
}
}
02:18:06.707 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
02:18:06.892 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52) [main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41) [main/:?]
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43) [main/:?]
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155) [main/:?]
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299) [main/:?]
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109) [main/:?]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) [junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
02:18:06.917 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
02:18:07.029 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskF8zSoX/test-tmp
02:18:07.030 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:18:07.030 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskF8zSoX/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:18:07.034 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
02:18:07.395 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:18:07.396 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:18:07.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:07.399380 13168 mini_oidc.cc:150] Starting JWKS server
02:18:07.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:07.403218 13168 webserver.cc:492] Webserver started at http://127.0.0.1:37799/ using document root <none> and password file <none>
02:18:07.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:07.403605 13168 mini_oidc.cc:186] Starting OIDC Discovery server
02:18:07.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:07.405301 13168 webserver.cc:492] Webserver started at http://127.0.0.1:39257/ using document root <none> and password file <none>
02:18:07.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
02:18:07.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/principal' for realm 'KRBTEST.COM',
02:18:07.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
02:18:07.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:07 dist-test-slave-df1k krb5kdc[13180](info): setting up network...
02:18:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
02:18:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:07 dist-test-slave-df1k krb5kdc[13180](info): set up 2 sockets
02:18:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:07 dist-test-slave-df1k krb5kdc[13180](info): commencing operation
02:18:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
02:18:07.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:07.653952 13158 heartbeater.cc:499] Master 127.12.37.252:34071 was elected leader, sending a full tablet report...
02:18:09.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:09.515880 13168 mini_kdc.cc:121] Time spent starting KDC: real 2.074s user 0.002s sys 0.019s
02:18:09.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
02:18:09.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:18:09.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
02:18:09.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
02:18:09.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:18:09.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
02:18:09.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
02:18:09.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:18:09.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
02:18:09.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:18:09.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/test-user.keytab.
02:18:09.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/test-user.keytab.
02:18:09.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:09 dist-test-slave-df1k krb5kdc[13180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185889, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:18:09.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:18:09.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-23T02:18:09Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:18:09.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-23T02:18:09Z Disabled control of system clock
02:18:09.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.220.62@KRBTEST.COM; defaulting to no policy
02:18:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:09.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.220.62@KRBTEST.COM" created.
02:18:09.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:09.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.220.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:09.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.220.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:09.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.12.220.62@KRBTEST.COM; defaulting to no policy
02:18:09.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:09.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.12.220.62@KRBTEST.COM" created.
02:18:09.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:09.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.220.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:09.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.220.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:09.743824 13168 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/wal
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/data
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/logs
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/data/info.pb
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:18:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:18:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.220.62:35095
02:18:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.220.62
02:18:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:18:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab
02:18:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.220.62
02:18:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:18:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:18:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:18:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:18:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.220.20:39563
02:18:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:18:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:18:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:18:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:18:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.220.62:35095,127.12.220.61:43219,127.12.220.60:46883
02:18:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:18:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37799/jwks.json
02:18:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:18:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:18:10.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:10.064702 13197 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:18:10.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:10.065373 13197 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:18:10.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:10.066116 13197 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:18:10.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:10.099057 13197 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:18:10.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:10.099321 13197 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:18:10.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:10.099702 13197 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:18:10.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:10.100008 13197 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:18:10.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:10.100286 13197 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:18:10.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:10.100574 13197 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:18:10.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:10.100811 13197 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37799/jwks.json
02:18:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:10.155041 13197 master_runner.cc:386] Master server non-default flags:
02:18:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:18:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.220.20:39563
02:18:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:18:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:18:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/data
02:18:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/wal
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.220.62:35095,127.12.220.61:43219,127.12.220.60:46883
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.220.62
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.220.62:35095
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37799/jwks.json
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/data/info.pb
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.220.62
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/logs
02:18:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:18:10.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:18:10.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:18:10.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:18:10.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:18:10.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 51f761c37259d8159142e890d4d8106f68bfa520
02:18:10.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:18:10.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 01:43:24 UTC on 5fd53c4cbb9d
02:18:10.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8673
02:18:10.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:18:10.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:10.156723 13197 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:18:10.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:10.158152 13197 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:18:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:10.170918 13203 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:12.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:10.172765 13204 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:12.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:11.573379 13202 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 13197
02:18:12.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:12.033440 13197 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.863s user 0.001s sys 0.001s
02:18:12.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:12.034230 13205 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1860 milliseconds
02:18:12.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:12.034451 13197 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.864s user 0.001s sys 0.002s
02:18:12.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:12.035997 13206 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:12.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.035962 13197 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:18:12.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.039448 13197 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:18:12.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.042265 13197 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:18:12.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.043732 13197 hybrid_clock.cc:648] HybridClock initialized: now 1761185892043665 us; error 62 us; skew 500 ppm
02:18:12.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.044726 13197 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:18:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:12 dist-test-slave-df1k krb5kdc[13180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185892, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:18:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.050704 13197 init.cc:377] Logged in from keytab as kudu/127.12.220.62@KRBTEST.COM (short username kudu)
02:18:12.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.055719 13197 webserver.cc:492] Webserver started at http://127.12.220.62:45557/ using document root <none> and password file <none>
02:18:12.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.056563 13197 fs_manager.cc:362] Metadata directory not provided
02:18:12.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.056743 13197 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:18:12.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.057116 13197 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:18:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.061431 13197 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/data/instance:
02:18:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c4b2eb94707a4d91bcf53023db1b2715"
02:18:12.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:12 on dist-test-slave-df1k"
02:18:12.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.062466 13197 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/wal/instance:
02:18:12.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c4b2eb94707a4d91bcf53023db1b2715"
02:18:12.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:12 on dist-test-slave-df1k"
02:18:12.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.070281 13197 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
02:18:12.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.076691 13216 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:18:12.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.078101 13197 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
02:18:12.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.078421 13197 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/data,/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/wal
02:18:12.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c4b2eb94707a4d91bcf53023db1b2715"
02:18:12.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:12 on dist-test-slave-df1k"
02:18:12.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.079017 13197 fs_report.cc:389] FS layout report
02:18:12.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:18:12.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/wal
02:18:12.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/wal
02:18:12.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/data/data
02:18:12.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:18:12.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:18:12.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:18:12.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:18:12.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:18:12.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:18:12.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:18:12.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:18:12.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:18:12.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:18:12.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:18:12.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.126420 13197 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:18:12.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.135568 13197 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:18:12.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.135978 13197 kserver.cc:163] Server-wide thread pool size limit: 3276
02:18:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.217916 13197 rpc_server.cc:307] RPC server started. Bound to: 127.12.220.62:35095
02:18:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.217978 13268 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.220.62:35095 every 8 connection(s)
02:18:12.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.220453 13197 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/data/info.pb
02:18:12.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.222787 13168 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu as pid 13197
02:18:12.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.223341 13168 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/wal/instance
02:18:12.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.228495 13269 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:18:12.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.220.61@KRBTEST.COM; defaulting to no policy
02:18:12.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.242265 13269 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } has no permanent_uuid. Determining permanent_uuid...
02:18:12.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:12.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.220.61@KRBTEST.COM" created.
02:18:12.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:12 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185892, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.62@KRBTEST.COM for kudu/127.12.220.62@KRBTEST.COM
02:18:12.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:12.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.220.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:12.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.220.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:12.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.282199 13269 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } has no permanent_uuid. Determining permanent_uuid...
02:18:12.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:12.284926 13218 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.220.61:43219: connect: Connection refused (error 111)
02:18:12.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.12.220.61@KRBTEST.COM; defaulting to no policy
02:18:12.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:12.290508 13269 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.220.61:43219: Network error: Client connection negotiation failed: client connection to 127.12.220.61:43219: connect: Connection refused (error 111)
02:18:12.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:12.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.12.220.61@KRBTEST.COM" created.
02:18:12.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:12.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.220.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:12.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.220.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.317273 13168 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:18:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:18:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/wal
02:18:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/data
02:18:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:18:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:18:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:18:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:18:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:18:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:18:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/logs
02:18:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/data/info.pb
02:18:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:18:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:18:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:18:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:18:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:18:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.220.61:43219
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.220.61
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.220.61
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.220.20:39563
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.220.62:35095,127.12.220.61:43219,127.12.220.60:46883
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37799/jwks.json
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:18:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:18:12.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.340023 13269 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } attempt: 1
02:18:12.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:12.343806 13269 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.220.61:43219: Network error: Client connection negotiation failed: client connection to 127.12.220.61:43219: connect: Connection refused (error 111)
02:18:12.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.412177 13269 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } attempt: 2
02:18:12.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:12.416716 13269 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.220.61:43219: Network error: Client connection negotiation failed: client connection to 127.12.220.61:43219: connect: Connection refused (error 111)
02:18:12.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.508077 13269 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } attempt: 3
02:18:12.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:12.512197 13269 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.220.61:43219: Network error: Client connection negotiation failed: client connection to 127.12.220.61:43219: connect: Connection refused (error 111)
02:18:12.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.655627 13269 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } attempt: 4
02:18:12.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:12.656983 13278 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:18:12.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:12.657603 13278 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:18:12.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:12.658316 13278 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:18:12.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:12.660028 13269 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.220.61:43219: Network error: Client connection negotiation failed: client connection to 127.12.220.61:43219: connect: Connection refused (error 111)
02:18:12.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:12.693059 13278 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:18:12.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:12.693351 13278 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:18:12.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:12.693624 13278 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:18:12.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:12.693818 13278 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:18:12.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:12.694016 13278 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:18:12.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:12.694207 13278 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:18:12.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:12.694402 13278 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37799/jwks.json
02:18:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.731921 13278 master_runner.cc:386] Master server non-default flags:
02:18:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:18:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.220.20:39563
02:18:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:18:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:18:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/data
02:18:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/wal
02:18:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:18:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:18:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.220.62:35095,127.12.220.61:43219,127.12.220.60:46883
02:18:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab
02:18:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.220.61
02:18:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:18:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:18:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:18:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.220.61:43219
02:18:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:18:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:18:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:18:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37799/jwks.json
02:18:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:18:12.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:18:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:18:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:18:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:18:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/data/info.pb
02:18:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.220.61
02:18:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:18:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:18:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:18:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:18:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:18:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:18:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/logs
02:18:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:18:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:18:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:18:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:18:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:18:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 51f761c37259d8159142e890d4d8106f68bfa520
02:18:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:18:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 01:43:24 UTC on 5fd53c4cbb9d
02:18:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8673
02:18:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:18:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.733281 13278 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:18:12.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.734247 13278 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:18:12.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:12.745335 13284 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:12.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:12.963492 13269 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } attempt: 5
02:18:12.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:12.970273 13269 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.220.61:43219: Network error: Client connection negotiation failed: client connection to 127.12.220.61:43219: connect: Connection refused (error 111)
02:18:13.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:13.517879 13269 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } attempt: 6
02:18:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:13.543812 13269 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.220.61:43219: Network error: Client connection negotiation failed: client connection to 127.12.220.61:43219: connect: Connection refused (error 111)
02:18:13.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:13.910014 13265 debug-util.cc:398] Leaking SignalData structure 0x7b0800071060 after lost signal to thread 13198
02:18:13.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:13.911073 13265 debug-util.cc:398] Leaking SignalData structure 0x7b08000717c0 after lost signal to thread 13268
02:18:14.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:12.747931 13285 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:14.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.035738 13287 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:14.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.038625 13286 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1286 milliseconds
02:18:14.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.038687 13278 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:18:14.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.039968 13278 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:18:14.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.043102 13278 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:18:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.044514 13278 hybrid_clock.cc:648] HybridClock initialized: now 1761185894044484 us; error 33 us; skew 500 ppm
02:18:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.045341 13278 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:18:14.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:14 dist-test-slave-df1k krb5kdc[13180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185894, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:18:14.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.051810 13278 init.cc:377] Logged in from keytab as kudu/127.12.220.61@KRBTEST.COM (short username kudu)
02:18:14.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.058454 13278 webserver.cc:492] Webserver started at http://127.12.220.61:45543/ using document root <none> and password file <none>
02:18:14.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.059526 13278 fs_manager.cc:362] Metadata directory not provided
02:18:14.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.059835 13278 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:18:14.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.060330 13278 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:18:14.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.065495 13278 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/data/instance:
02:18:14.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f253219e08f4cbea557cc68a0603a85"
02:18:14.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:14 on dist-test-slave-df1k"
02:18:14.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.066654 13278 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/wal/instance:
02:18:14.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f253219e08f4cbea557cc68a0603a85"
02:18:14.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:14 on dist-test-slave-df1k"
02:18:14.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.076768 13278 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.000s sys 0.009s
02:18:14.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.085371 13298 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:18:14.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.086565 13278 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
02:18:14.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.086915 13278 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/data,/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/wal
02:18:14.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f253219e08f4cbea557cc68a0603a85"
02:18:14.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:14 on dist-test-slave-df1k"
02:18:14.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.087301 13278 fs_report.cc:389] FS layout report
02:18:14.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:18:14.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/wal
02:18:14.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/wal
02:18:14.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/data/data
02:18:14.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:18:14.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:18:14.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:18:14.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:18:14.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:18:14.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:18:14.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:18:14.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:18:14.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:18:14.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:18:14.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:18:14.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.142031 13278 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:18:14.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.147943 13278 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:18:14.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.148473 13278 kserver.cc:163] Server-wide thread pool size limit: 3276
02:18:14.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.261145 13278 rpc_server.cc:307] RPC server started. Bound to: 127.12.220.61:43219
02:18:14.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.261338 13349 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.220.61:43219 every 8 connection(s)
02:18:14.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.264163 13278 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/data/info.pb
02:18:14.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.271993 13168 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu as pid 13278
02:18:14.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.272467 13168 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/wal/instance
02:18:14.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.273552 13350 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:18:14.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.220.60@KRBTEST.COM; defaulting to no policy
02:18:14.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.292402 13350 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } has no permanent_uuid. Determining permanent_uuid...
02:18:14.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:14.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.220.60@KRBTEST.COM" created.
02:18:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:14 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185894, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.61@KRBTEST.COM for kudu/127.12.220.62@KRBTEST.COM
02:18:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.331992 13350 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } has no permanent_uuid. Determining permanent_uuid...
02:18:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.220.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:14.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.220.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:14.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:14 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185894, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.61@KRBTEST.COM for kudu/127.12.220.61@KRBTEST.COM
02:18:14.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.12.220.60@KRBTEST.COM; defaulting to no policy
02:18:14.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:14.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.12.220.60@KRBTEST.COM" created.
02:18:14.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.387800 13350 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } has no permanent_uuid. Determining permanent_uuid...
02:18:14.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.390532 13300 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.220.60:46883: connect: Connection refused (error 111)
02:18:14.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:14.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.220.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:14.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.220.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:14.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.395756 13350 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.220.60:46883: Network error: Client connection negotiation failed: client connection to 127.12.220.60:46883: connect: Connection refused (error 111)
02:18:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.397670 13168 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:18:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:18:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/wal
02:18:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/data
02:18:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:18:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:18:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:18:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:18:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:18:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:18:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/logs
02:18:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/data/info.pb
02:18:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:18:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:18:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:18:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:18:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:18:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:18:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:18:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:18:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:18:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:18:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:18:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:18:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.220.60:46883
02:18:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.220.60
02:18:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:18:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab
02:18:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.220.60
02:18:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:18:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:18:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:18:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:18:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.220.20:39563
02:18:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:18:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:18:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:18:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:18:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.220.62:35095,127.12.220.61:43219,127.12.220.60:46883
02:18:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:18:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37799/jwks.json
02:18:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:18:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:18:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.445297 13350 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } attempt: 1
02:18:14.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.451259 13350 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.220.60:46883: Network error: Client connection negotiation failed: client connection to 127.12.220.60:46883: connect: Connection refused (error 111)
02:18:14.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.519766 13350 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } attempt: 2
02:18:14.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.523905 13350 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.220.60:46883: Network error: Client connection negotiation failed: client connection to 127.12.220.60:46883: connect: Connection refused (error 111)
02:18:14.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.604442 13269 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } attempt: 7
02:18:14.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.615327 13350 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } attempt: 3
02:18:14.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:14 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185892, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.62@KRBTEST.COM for kudu/127.12.220.61@KRBTEST.COM
02:18:14.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.620529 13350 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.220.60:46883: Network error: Client connection negotiation failed: client connection to 127.12.220.60:46883: connect: Connection refused (error 111)
02:18:14.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.631453 13269 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } has no permanent_uuid. Determining permanent_uuid...
02:18:14.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.637912 13269 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.220.60:46883: Network error: Client connection negotiation failed: client connection to 127.12.220.60:46883: connect: Connection refused (error 111)
02:18:14.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.696337 13269 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } attempt: 1
02:18:14.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.701391 13269 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.220.60:46883: Network error: Client connection negotiation failed: client connection to 127.12.220.60:46883: connect: Connection refused (error 111)
02:18:14.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.763974 13350 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } attempt: 4
02:18:14.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.768513 13350 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.220.60:46883: Network error: Client connection negotiation failed: client connection to 127.12.220.60:46883: connect: Connection refused (error 111)
02:18:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.782944 13269 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } attempt: 2
02:18:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.789844 13269 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.220.60:46883: Network error: Client connection negotiation failed: client connection to 127.12.220.60:46883: connect: Connection refused (error 111)
02:18:14.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.841849 13360 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:18:14.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.842694 13360 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:18:14.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.844019 13360 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:18:14.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.875344 13269 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } attempt: 3
02:18:14.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.881624 13269 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.220.60:46883: Network error: Client connection negotiation failed: client connection to 127.12.220.60:46883: connect: Connection refused (error 111)
02:18:14.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.887476 13360 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:18:14.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.887729 13360 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:18:14.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.888111 13360 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:18:14.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.888407 13360 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:18:14.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.888685 13360 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:18:14.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.888969 13360 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:18:14.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.889189 13360 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37799/jwks.json
02:18:14.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.930574 13360 master_runner.cc:386] Master server non-default flags:
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.220.20:39563
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/data
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/wal
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.220.62:35095,127.12.220.61:43219,127.12.220.60:46883
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.220.60
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.220.60:46883
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37799/jwks.json
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/data/info.pb
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.220.60
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:18:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:18:14.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:18:14.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:18:14.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:18:14.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/logs
02:18:14.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:18:14.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:18:14.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:18:14.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:18:14.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:18:14.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 51f761c37259d8159142e890d4d8106f68bfa520
02:18:14.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:18:14.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 01:43:24 UTC on 5fd53c4cbb9d
02:18:14.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8673
02:18:14.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:18:14.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.932173 13360 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:18:14.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:14.933511 13360 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:18:14.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.953013 13369 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:15.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:15.022173 13269 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } attempt: 4
02:18:15.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:15.030551 13269 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.220.60:46883: Network error: Client connection negotiation failed: client connection to 127.12.220.60:46883: connect: Connection refused (error 111)
02:18:15.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:15.067998 13350 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } attempt: 5
02:18:15.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:15.074594 13350 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.220.60:46883: Network error: Client connection negotiation failed: client connection to 127.12.220.60:46883: connect: Connection refused (error 111)
02:18:15.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:15.314134 13269 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } attempt: 5
02:18:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:15.322283 13269 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.220.60:46883: Network error: Client connection negotiation failed: client connection to 127.12.220.60:46883: connect: Connection refused (error 111)
02:18:15.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:15.622026 13350 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } attempt: 6
02:18:15.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:15.659052 13350 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.220.60:46883: Network error: Client connection negotiation failed: client connection to 127.12.220.60:46883: connect: Connection refused (error 111)
02:18:15.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:15.874905 13269 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } attempt: 6
02:18:15.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:15.926065 13269 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.220.60:46883: Network error: Client connection negotiation failed: client connection to 127.12.220.60:46883: connect: Connection refused (error 111)
02:18:16.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:16.719655 13350 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } attempt: 7
02:18:16.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:16.748234 13350 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.220.60:46883: Network error: Client connection negotiation failed: client connection to 127.12.220.60:46883: connect: Connection refused (error 111)
02:18:16.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:16.350342 13368 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 13360
02:18:16.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:14.954295 13370 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:16.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:16.883816 13360 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.933s user 0.001s sys 0.001s
02:18:16.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:16.884155 13360 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.934s user 0.002s sys 0.001s
02:18:16.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:16.884696 13371 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1932 milliseconds
02:18:16.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:16.890798 13372 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:16.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:16.890842 13360 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:18:16.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:16.892204 13360 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:18:16.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:16.894668 13360 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:18:16.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:16.896001 13360 hybrid_clock.cc:648] HybridClock initialized: now 1761185896895969 us; error 36 us; skew 500 ppm
02:18:16.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:16.896754 13360 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:18:16.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:16 dist-test-slave-df1k krb5kdc[13180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185896, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:18:16.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:16.902799 13360 init.cc:377] Logged in from keytab as kudu/127.12.220.60@KRBTEST.COM (short username kudu)
02:18:16.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:16.908269 13360 webserver.cc:492] Webserver started at http://127.12.220.60:46715/ using document root <none> and password file <none>
02:18:16.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:16.909358 13360 fs_manager.cc:362] Metadata directory not provided
02:18:16.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:16.909574 13360 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:18:16.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:16.910058 13360 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:18:16.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:16.915100 13360 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/data/instance:
02:18:16.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "423a10751bbd43fda4725dd498e20dda"
02:18:16.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:16 on dist-test-slave-df1k"
02:18:16.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:16.916113 13360 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/wal/instance:
02:18:16.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "423a10751bbd43fda4725dd498e20dda"
02:18:16.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:16 on dist-test-slave-df1k"
02:18:16.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:16.924552 13360 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.001s sys 0.008s
02:18:16.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:16.930255 13387 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:18:16.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:16.931409 13360 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
02:18:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:16.931738 13360 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/data,/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/wal
02:18:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "423a10751bbd43fda4725dd498e20dda"
02:18:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:16 on dist-test-slave-df1k"
02:18:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:16.932042 13360 fs_report.cc:389] FS layout report
02:18:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:18:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/wal
02:18:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/wal
02:18:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/data/data
02:18:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:18:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:18:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:18:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:18:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:18:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:18:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:18:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:18:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:18:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:18:16.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:18:16.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:16.973382 13269 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } attempt: 7
02:18:16.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:16.978960 13269 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.220.60:46883: Network error: Client connection negotiation failed: client connection to 127.12.220.60:46883: connect: Connection refused (error 111)
02:18:16.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:16.991354 13360 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:18:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:16.997174 13360 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:18:16.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:16.998191 13360 kserver.cc:163] Server-wide thread pool size limit: 3276
02:18:17.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.085362 13360 rpc_server.cc:307] RPC server started. Bound to: 127.12.220.60:46883
02:18:17.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.085425 13440 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.220.60:46883 every 8 connection(s)
02:18:17.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.089033 13360 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/data/info.pb
02:18:17.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.092515 13168 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu as pid 13360
02:18:17.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.092976 13168 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/wal/instance
02:18:17.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.097945 13441 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:18:17.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.118254 13441 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } has no permanent_uuid. Determining permanent_uuid...
02:18:17.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:17 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185889, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.12.220.62@KRBTEST.COM
02:18:17.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:17 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185896, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.60@KRBTEST.COM for kudu/127.12.220.62@KRBTEST.COM
02:18:17.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:17 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185889, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.12.220.61@KRBTEST.COM
02:18:17.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.156380 13441 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } has no permanent_uuid. Determining permanent_uuid...
02:18:17.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:17 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185896, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.60@KRBTEST.COM for kudu/127.12.220.61@KRBTEST.COM
02:18:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:17 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185889, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.12.220.60@KRBTEST.COM
02:18:17.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.180235 13441 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } has no permanent_uuid. Determining permanent_uuid...
02:18:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:17 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185896, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.60@KRBTEST.COM for kudu/127.12.220.60@KRBTEST.COM
02:18:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.227102 13441 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda: Bootstrap starting.
02:18:17.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.241824 13441 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda: Neither blocks nor log segments found. Creating new log.
02:18:17.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.245479 13441 log.cc:826] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda: Log is configured to *not* fsync() on all Append() calls
02:18:17.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.251019 13441 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda: No bootstrap required, opened a new log
02:18:17.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.291288 13441 raft_consensus.cc:359] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } }
02:18:17.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.292733 13441 raft_consensus.cc:385] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:18:17.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.293182 13441 raft_consensus.cc:740] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 423a10751bbd43fda4725dd498e20dda, State: Initialized, Role: FOLLOWER
02:18:17.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.294610 13441 consensus_queue.cc:260] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } }
02:18:17.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.302044 13452 sys_catalog.cc:455] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } } }
02:18:17.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.303081 13452 sys_catalog.cc:458] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [sys.catalog]: This master's current role is: FOLLOWER
02:18:17.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.304908 13441 sys_catalog.cc:565] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [sys.catalog]: configured and running, proceeding with master startup.
02:18:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.338208 13452 raft_consensus.cc:493] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:18:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.338781 13452 raft_consensus.cc:515] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } }
02:18:17.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.341475 13452 leader_election.cc:290] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c4b2eb94707a4d91bcf53023db1b2715 (127.12.220.62:35095), 9f253219e08f4cbea557cc68a0603a85 (127.12.220.61:43219)
02:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:17.349427 13464 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:18:17.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:17.349746 13464 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:18:17.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:17.357291 13265 debug-util.cc:398] Leaking SignalData structure 0x7b0800071f40 after lost signal to thread 13198
02:18:17.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.340757 13360 master_runner.cc:186] Error getting master registration for 127.12.220.62:35095: OK, instance_id { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" instance_seqno: 1761185892194412 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:18:17.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:17.374264 13360 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:18:17.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:17.503835 13395 tablet.cc:2378] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:18:18.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:18.353703 13464 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:18:18.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:18.354044 13464 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:18:18.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.838749 13350 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } attempt: 8
02:18:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:18 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185894, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.61@KRBTEST.COM for kudu/127.12.220.60@KRBTEST.COM
02:18:18.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.879601 13350 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85: Bootstrap starting.
02:18:18.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.888238 13350 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85: Neither blocks nor log segments found. Creating new log.
02:18:18.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.891263 13350 log.cc:826] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85: Log is configured to *not* fsync() on all Append() calls
02:18:18.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.897342 13350 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85: No bootstrap required, opened a new log
02:18:18.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.928447 13350 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } }
02:18:18.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.929293 13350 raft_consensus.cc:385] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:18:18.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.929595 13350 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9f253219e08f4cbea557cc68a0603a85, State: Initialized, Role: FOLLOWER
02:18:18.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.930483 13350 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } }
02:18:18.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.933518 13470 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } } }
02:18:18.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.934296 13470 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [sys.catalog]: This master's current role is: FOLLOWER
02:18:18.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.935928 13350 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [sys.catalog]: configured and running, proceeding with master startup.
02:18:18.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.936728 13325 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "423a10751bbd43fda4725dd498e20dda" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9f253219e08f4cbea557cc68a0603a85" is_pre_election: true
02:18:18.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.937605 13325 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 423a10751bbd43fda4725dd498e20dda in term 0.
02:18:18.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:18.938293 13325 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:37714 (request call id 1) took 1594 ms (client timeout 2002 ms). Trace:
02:18:18.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:18.938681 13325 rpcz_store.cc:269] 1023 02:18:17.343561 (+ 0us) service_pool.cc:168] Inserting onto call queue
02:18:18.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:17.343750 (+ 189us) service_pool.cc:225] Handling call
02:18:18.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:18.938228 (+1594478us) inbound_call.cc:173] Queueing success response
02:18:18.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
02:18:18.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.940542 13389 leader_election.cc:304] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 423a10751bbd43fda4725dd498e20dda, 9f253219e08f4cbea557cc68a0603a85; no voters:
02:18:18.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.953353 13473 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [term 0 FOLLOWER]: Leader pre-election won for term 1
02:18:18.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.953814 13473 raft_consensus.cc:493] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:18:18.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.954139 13473 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [term 0 FOLLOWER]: Advancing to term 1
02:18:18.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.963027 13473 raft_consensus.cc:515] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } }
02:18:18.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.967056 13325 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "423a10751bbd43fda4725dd498e20dda" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9f253219e08f4cbea557cc68a0603a85"
02:18:18.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.967634 13325 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [term 0 FOLLOWER]: Advancing to term 1
02:18:18.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.976956 13325 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 423a10751bbd43fda4725dd498e20dda in term 1.
02:18:18.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.966439 13473 leader_election.cc:290] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [CANDIDATE]: Term 1 election: Requested vote from peers c4b2eb94707a4d91bcf53023db1b2715 (127.12.220.62:35095), 9f253219e08f4cbea557cc68a0603a85 (127.12.220.61:43219)
02:18:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.978140 13389 leader_election.cc:304] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 423a10751bbd43fda4725dd498e20dda, 9f253219e08f4cbea557cc68a0603a85; no voters:
02:18:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.979065 13473 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [term 1 FOLLOWER]: Leader election won for term 1
02:18:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.981858 13473 raft_consensus.cc:697] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [term 1 LEADER]: Becoming Leader. State: Replica: 423a10751bbd43fda4725dd498e20dda, State: Running, Role: LEADER
02:18:18.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:18.983120 13473 consensus_queue.cc:237] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } }
02:18:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.008782 13479 sys_catalog.cc:455] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [sys.catalog]: SysCatalogTable state changed. Reason: New leader 423a10751bbd43fda4725dd498e20dda. Latest consensus state: current_term: 1 leader_uuid: "423a10751bbd43fda4725dd498e20dda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } } }
02:18:19.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.009895 13479 sys_catalog.cc:458] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [sys.catalog]: This master's current role is: LEADER
02:18:19.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.012719 13485 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
02:18:19.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.028337 13485 catalog_manager.cc:1494] Initializing Kudu cluster ID...
02:18:19.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:19.035333 13484 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:18:19.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:19.035708 13484 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:18:19.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.040592 13269 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } attempt: 8
02:18:19.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.061029 13325 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [term 1 FOLLOWER]: Refusing update from remote peer 423a10751bbd43fda4725dd498e20dda: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:18:19.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.063503 13473 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [LEADER]: Connected to new peer: Peer: permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:18:19.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.015291 13278 master_runner.cc:186] Error getting master registration for 127.12.220.62:35095: OK, instance_id { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" instance_seqno: 1761185892194412 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:18:19.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.097335 13278 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:18:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:19 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185892, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.62@KRBTEST.COM for kudu/127.12.220.60@KRBTEST.COM
02:18:19.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.153729 13269 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715: Bootstrap starting.
02:18:19.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.156293 13470 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 423a10751bbd43fda4725dd498e20dda. Latest consensus state: current_term: 1 leader_uuid: "423a10751bbd43fda4725dd498e20dda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } } }
02:18:19.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:19.161053 13304 tablet.cc:2378] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:18:19.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.157058 13470 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [sys.catalog]: This master's current role is: FOLLOWER
02:18:19.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.169672 13269 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715: Neither blocks nor log segments found. Creating new log.
02:18:19.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.172358 13269 log.cc:826] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715: Log is configured to *not* fsync() on all Append() calls
02:18:19.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.185464 13488 mvcc.cc:204] Tried to move back new op lower bound from 7213817442537861120 to 7213817442245713920. Current Snapshot: MvccSnapshot[applied={T|T < 7213817442537861120}]
02:18:19.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.198999 13479 sys_catalog.cc:455] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "423a10751bbd43fda4725dd498e20dda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } } }
02:18:19.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.200374 13479 sys_catalog.cc:458] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [sys.catalog]: This master's current role is: LEADER
02:18:19.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.210422 13470 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "423a10751bbd43fda4725dd498e20dda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } } }
02:18:19.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.211159 13470 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [sys.catalog]: This master's current role is: FOLLOWER
02:18:19.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.214241 13269 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715: No bootstrap required, opened a new log
02:18:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.219164 13485 catalog_manager.cc:1357] Generated new cluster ID: d3842f9e2fed40dbad6ac30b3c3a89e0
02:18:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.219473 13485 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
02:18:19.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.301394 13269 raft_consensus.cc:359] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } }
02:18:19.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.304996 13269 raft_consensus.cc:385] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:18:19.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.305593 13269 raft_consensus.cc:740] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c4b2eb94707a4d91bcf53023db1b2715, State: Initialized, Role: FOLLOWER
02:18:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.306780 13269 consensus_queue.cc:260] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } }
02:18:19.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.321344 13494 sys_catalog.cc:455] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } } }
02:18:19.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.329505 13494 sys_catalog.cc:458] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [sys.catalog]: This master's current role is: FOLLOWER
02:18:19.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.326224 13243 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "423a10751bbd43fda4725dd498e20dda" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c4b2eb94707a4d91bcf53023db1b2715"
02:18:19.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.326821 13244 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "423a10751bbd43fda4725dd498e20dda" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c4b2eb94707a4d91bcf53023db1b2715" is_pre_election: true
02:18:19.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:19.332129 13244 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:40974 (request call id 2) took 1986 ms (client timeout 2002 ms). Trace:
02:18:19.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:19.332464 13244 rpcz_store.cc:269] 1023 02:18:17.345864 (+ 0us) service_pool.cc:168] Inserting onto call queue
02:18:19.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:17.345990 (+ 126us) service_pool.cc:225] Handling call
02:18:19.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:19.332056 (+1986066us) inbound_call.cc:173] Queueing success response
02:18:19.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
02:18:19.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.327759 13242 raft_consensus.cc:3060] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [term 0 FOLLOWER]: Advancing to term 1
02:18:19.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.325986 13269 sys_catalog.cc:565] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [sys.catalog]: configured and running, proceeding with master startup.
02:18:19.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.344170 13242 raft_consensus.cc:1275] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [term 1 FOLLOWER]: Refusing update from remote peer 423a10751bbd43fda4725dd498e20dda: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:18:19.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.346045 13473 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [LEADER]: Connected to new peer: Peer: permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
02:18:19.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.384196 13485 catalog_manager.cc:1380] Generated new certificate authority record
02:18:19.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.392585 13485 catalog_manager.cc:1514] Loading token signing keys...
02:18:19.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:19.401438 13222 tablet.cc:2378] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:18:19.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.450177 13473 sys_catalog.cc:455] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "423a10751bbd43fda4725dd498e20dda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } } }
02:18:19.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.450866 13473 sys_catalog.cc:458] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [sys.catalog]: This master's current role is: LEADER
02:18:19.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.450968 13501 sys_catalog.cc:455] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "423a10751bbd43fda4725dd498e20dda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } } }
02:18:19.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.454005 13501 sys_catalog.cc:458] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [sys.catalog]: This master's current role is: FOLLOWER
02:18:19.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.452953 13494 sys_catalog.cc:455] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 423a10751bbd43fda4725dd498e20dda. Latest consensus state: current_term: 1 leader_uuid: "423a10751bbd43fda4725dd498e20dda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } } }
02:18:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.454505 13494 sys_catalog.cc:458] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [sys.catalog]: This master's current role is: FOLLOWER
02:18:19.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.485476 13485 catalog_manager.cc:6022] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda: Generated new TSK 0
02:18:19.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.486593 13485 catalog_manager.cc:1524] Initializing in-progress tserver states...
02:18:19.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.538722 13511 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:18:19.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.220.1@KRBTEST.COM; defaulting to no policy
02:18:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.220.1@KRBTEST.COM" created.
02:18:19.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:19.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.220.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:19.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.220.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:19.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:19.630379 13168 mini_kdc.cc:279] Time spent creating service keytab for kudu/127.12.220.1: real 0.125s user 0.003s sys 0.015s
02:18:19.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.12.220.1@KRBTEST.COM; defaulting to no policy
02:18:19.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:19.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.12.220.1@KRBTEST.COM" created.
02:18:19.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:19.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.220.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:19.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.220.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:19.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:19.678385 13168 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:18:19.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:18:19.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-0/wal
02:18:19.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-0/data
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-0/logs
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-0/data/info.pb
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.220.1:0
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.220.1
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.220.1
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.220.62:35095,127.12.220.61:43219,127.12.220.60:46883
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.220.1
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:18:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:18:19.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:18:19.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:18:19.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.220.20:39563
02:18:19.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:18:19.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:18:19.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:18:19.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:18:20.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:20.043131 13484 catalog_manager.cc:1269] Loaded cluster ID: d3842f9e2fed40dbad6ac30b3c3a89e0
02:18:20.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:20.047781 13484 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85: loading cluster ID for follower catalog manager: success
02:18:20.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:20.064396 13484 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85: acquiring CA information for follower catalog manager: success
02:18:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:20.088932 13484 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:18:20.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:20.167753 13515 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:18:20.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:20.168457 13515 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:18:20.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:20.169528 13515 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:18:20.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:20.222384 13515 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:18:20.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:20.223886 13515 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.220.1
02:18:20.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:20.276386 13515 tablet_server_runner.cc:78] Tablet server non-default flags:
02:18:20.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:18:20.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.220.20:39563
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-0/data
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-0/wal
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.220.1
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.220.1:0
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-0/data/info.pb
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.220.1
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.220.62:35095,127.12.220.61:43219,127.12.220.60:46883
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.220.1
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-0/logs
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:18:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:18:20.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:18:20.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:18:20.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 51f761c37259d8159142e890d4d8106f68bfa520
02:18:20.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:18:20.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 01:43:24 UTC on 5fd53c4cbb9d
02:18:20.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8673
02:18:20.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:18:20.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:20.278079 13515 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:18:20.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:20.279480 13515 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:18:20.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:20.304667 13522 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:20.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:20.585712 13511 catalog_manager.cc:1269] Loaded cluster ID: d3842f9e2fed40dbad6ac30b3c3a89e0
02:18:20.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:20.586551 13511 catalog_manager.cc:1562] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715: loading cluster ID for follower catalog manager: success
02:18:20.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:20.624367 13511 catalog_manager.cc:1584] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715: acquiring CA information for follower catalog manager: success
02:18:20.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:20.651908 13511 catalog_manager.cc:1612] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:18:21.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:21.706092 13521 debug-util.cc:398] Leaking SignalData structure 0x7b080002ee00 after lost signal to thread 13515
02:18:21.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:21.907819 13521 kernel_stack_watchdog.cc:198] Thread 13515 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 389ms:
02:18:21.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:18:21.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:18:21.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:18:21.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:18:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:20.306787 13523 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:21.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:21.914302 13524 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1603 milliseconds
02:18:21.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:21.916302 13515 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.601s user 0.000s sys 0.007s
02:18:21.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:21.916635 13515 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.602s user 0.000s sys 0.007s
02:18:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:21.918010 13515 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:18:21.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:21.918589 13526 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:21.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:21.920833 13515 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:18:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:21.933452 13515 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:18:21.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:21.935317 13515 hybrid_clock.cc:648] HybridClock initialized: now 1761185901935237 us; error 77 us; skew 500 ppm
02:18:21.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:21.936585 13515 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:18:21.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:21 dist-test-slave-df1k krb5kdc[13180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185901, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:18:21.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:21.945305 13515 init.cc:377] Logged in from keytab as kudu/127.12.220.1@KRBTEST.COM (short username kudu)
02:18:21.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:21.980289 13515 webserver.cc:492] Webserver started at http://127.12.220.1:45855/ using document root <none> and password file <none>
02:18:21.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:21.982244 13515 fs_manager.cc:362] Metadata directory not provided
02:18:21.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:21.982854 13515 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:18:21.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:21.983587 13515 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:18:22.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.003240 13515 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-0/data/instance:
02:18:22.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9b1513b94be14b8e96e0dbc87c9aada4"
02:18:22.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:21 on dist-test-slave-df1k"
02:18:22.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.005292 13515 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-0/wal/instance:
02:18:22.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9b1513b94be14b8e96e0dbc87c9aada4"
02:18:22.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:21 on dist-test-slave-df1k"
02:18:22.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.023487 13515 fs_manager.cc:696] Time spent creating directory manager: real 0.015s user 0.006s sys 0.008s
02:18:22.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.036834 13536 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:18:22.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.038672 13515 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.006s sys 0.004s
02:18:22.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.039273 13515 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-0/data,/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-0/wal
02:18:22.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9b1513b94be14b8e96e0dbc87c9aada4"
02:18:22.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:21 on dist-test-slave-df1k"
02:18:22.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.039927 13515 fs_report.cc:389] FS layout report
02:18:22.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:18:22.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-0/wal
02:18:22.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-0/wal
02:18:22.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-0/data/data
02:18:22.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:18:22.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:18:22.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:18:22.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:18:22.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:18:22.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:18:22.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:18:22.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:18:22.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:18:22.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:18:22.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:18:22.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:22 dist-test-slave-df1k krb5kdc[13180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185902, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:18:22.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.056975 13212 init.cc:295] Successfully reacquired a new kerberos TGT
02:18:22.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.136785 13515 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:18:22.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.148304 13515 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:18:22.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.148891 13515 kserver.cc:163] Server-wide thread pool size limit: 3276
02:18:22.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.158751 13515 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:18:22.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.171408 13515 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:18:22.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.171726 13515 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:18:22.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.172027 13515 ts_tablet_manager.cc:616] Registered 0 tablets
02:18:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.172261 13515 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:18:22.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.671459 13515 rpc_server.cc:307] RPC server started. Bound to: 127.12.220.1:42297
02:18:22.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.671556 13650 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.220.1:42297 every 8 connection(s)
02:18:22.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.674417 13515 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-0/data/info.pb
02:18:22.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.685055 13168 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu as pid 13515
02:18:22.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.685674 13168 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-0/wal/instance
02:18:22.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:22 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185901, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.1@KRBTEST.COM for kudu/127.12.220.61@KRBTEST.COM
02:18:22.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.220.2@KRBTEST.COM; defaulting to no policy
02:18:22.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:22 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185901, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.1@KRBTEST.COM for kudu/127.12.220.60@KRBTEST.COM
02:18:22.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:22.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.220.2@KRBTEST.COM" created.
02:18:22.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.777424 13651 heartbeater.cc:344] Connected to a master server at 127.12.220.60:46883
02:18:22.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.777968 13651 heartbeater.cc:461] Registering TS with master...
02:18:22.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.779359 13651 heartbeater.cc:507] Master 127.12.220.60:46883 requested a full tablet report, sending...
02:18:22.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.782588 13406 ts_manager.cc:194] Registered new tserver with Master: 9b1513b94be14b8e96e0dbc87c9aada4 (127.12.220.1:42297)
02:18:22.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:22 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185901, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.1@KRBTEST.COM for kudu/127.12.220.62@KRBTEST.COM
02:18:22.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.785167 13406 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.12.220.1@KRBTEST.COM'} at 127.12.220.1:50675
02:18:22.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.794314 13652 heartbeater.cc:344] Connected to a master server at 127.12.220.61:43219
02:18:22.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.794739 13652 heartbeater.cc:461] Registering TS with master...
02:18:22.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.795485 13652 heartbeater.cc:507] Master 127.12.220.61:43219 requested a full tablet report, sending...
02:18:22.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.799770 13315 ts_manager.cc:194] Registered new tserver with Master: 9b1513b94be14b8e96e0dbc87c9aada4 (127.12.220.1:42297)
02:18:22.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.811806 13656 heartbeater.cc:344] Connected to a master server at 127.12.220.62:35095
02:18:22.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.812197 13656 heartbeater.cc:461] Registering TS with master...
02:18:22.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.812814 13656 heartbeater.cc:507] Master 127.12.220.62:35095 requested a full tablet report, sending...
02:18:22.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:22.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.220.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:22.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.220.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:22.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.815649 13234 ts_manager.cc:194] Registered new tserver with Master: 9b1513b94be14b8e96e0dbc87c9aada4 (127.12.220.1:42297)
02:18:22.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:22.816449 13168 mini_kdc.cc:279] Time spent creating service keytab for kudu/127.12.220.2: real 0.113s user 0.003s sys 0.013s
02:18:22.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.12.220.2@KRBTEST.COM; defaulting to no policy
02:18:22.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:22.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.12.220.2@KRBTEST.COM" created.
02:18:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.220.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.220.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:22.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:22.857203 13168 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:18:22.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:18:22.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-1/wal
02:18:22.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-1/data
02:18:22.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:18:22.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:18:22.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:18:22.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:18:22.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:18:22.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:18:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-1/logs
02:18:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-1/data/info.pb
02:18:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:18:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:18:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:18:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:18:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:18:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:18:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:18:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:18:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:18:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:18:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.220.2:0
02:18:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.220.2
02:18:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.220.2
02:18:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:18:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.220.62:35095,127.12.220.61:43219,127.12.220.60:46883
02:18:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab
02:18:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.220.2
02:18:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:18:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:18:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:18:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:18:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.220.20:39563
02:18:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:18:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:18:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:18:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:18:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:23.292009 13665 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:18:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:23.293058 13665 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:18:23.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:23.294286 13665 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:18:23.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:23.337668 13665 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:18:23.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:23.338639 13665 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.220.2
02:18:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:23.398777 13665 tablet_server_runner.cc:78] Tablet server non-default flags:
02:18:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:18:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.220.20:39563
02:18:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:18:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:18:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-1/data
02:18:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-1/wal
02:18:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab
02:18:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.220.2
02:18:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:18:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:18:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.220.2:0
02:18:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:18:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:18:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:18:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:18:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:18:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:18:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-1/data/info.pb
02:18:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.220.2
02:18:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:18:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.220.62:35095,127.12.220.61:43219,127.12.220.60:46883
02:18:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:18:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:18:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:18:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:18:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:18:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.220.2
02:18:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-1/logs
02:18:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:18:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:18:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:18:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:18:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:18:23.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 51f761c37259d8159142e890d4d8106f68bfa520
02:18:23.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:18:23.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 01:43:24 UTC on 5fd53c4cbb9d
02:18:23.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8673
02:18:23.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:18:23.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:23.400329 13665 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:18:23.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:23.401650 13665 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:18:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:23.415428 13672 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:23.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:23.789453 13651 heartbeater.cc:499] Master 127.12.220.60:46883 was elected leader, sending a full tablet report...
02:18:24.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:24 dist-test-slave-df1k krb5kdc[13180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185904, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:18:24.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:24.058478 13294 init.cc:295] Successfully reacquired a new kerberos TGT
02:18:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:24.819608 13671 debug-util.cc:398] Leaking SignalData structure 0x7b080002ee00 after lost signal to thread 13665
02:18:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:25.656002 13671 kernel_stack_watchdog.cc:198] Thread 13665 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
02:18:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:18:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:18:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:18:25.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:18:25.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:23.418166 13673 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:25.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:25.657197 13665 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.240s user 0.000s sys 0.002s
02:18:25.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:25.657593 13665 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.241s user 0.000s sys 0.003s
02:18:25.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:25.659332 13675 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:25.662900 13674 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2240 milliseconds
02:18:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.662978 13665 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:18:25.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.664273 13665 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:18:25.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.666303 13665 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:18:25.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.667641 13665 hybrid_clock.cc:648] HybridClock initialized: now 1761185905667616 us; error 35 us; skew 500 ppm
02:18:25.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.668404 13665 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:18:25.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:25 dist-test-slave-df1k krb5kdc[13180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185905, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:18:25.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.674819 13665 init.cc:377] Logged in from keytab as kudu/127.12.220.2@KRBTEST.COM (short username kudu)
02:18:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.680478 13665 webserver.cc:492] Webserver started at http://127.12.220.2:44627/ using document root <none> and password file <none>
02:18:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.681821 13665 fs_manager.cc:362] Metadata directory not provided
02:18:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.682147 13665 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:18:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.682626 13665 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:18:25.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.686718 13665 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-1/data/instance:
02:18:25.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b771fc2735a44c4dbe39fa8aa208e045"
02:18:25.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:25 on dist-test-slave-df1k"
02:18:25.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.688042 13665 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-1/wal/instance:
02:18:25.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b771fc2735a44c4dbe39fa8aa208e045"
02:18:25.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:25 on dist-test-slave-df1k"
02:18:25.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.695400 13665 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
02:18:25.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.700708 13688 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:18:25.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.701874 13665 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
02:18:25.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.702181 13665 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-1/data,/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-1/wal
02:18:25.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b771fc2735a44c4dbe39fa8aa208e045"
02:18:25.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:25 on dist-test-slave-df1k"
02:18:25.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.702472 13665 fs_report.cc:389] FS layout report
02:18:25.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:18:25.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-1/wal
02:18:25.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-1/wal
02:18:25.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-1/data/data
02:18:25.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:18:25.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:18:25.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:18:25.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:18:25.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:18:25.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:18:25.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:18:25.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:18:25.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:18:25.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:18:25.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:18:25.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.758191 13665 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:18:25.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.763628 13665 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:18:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.764047 13665 kserver.cc:163] Server-wide thread pool size limit: 3276
02:18:25.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.767445 13665 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:18:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.771550 13665 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:18:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.771744 13665 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:18:25.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.771941 13665 ts_tablet_manager.cc:616] Registered 0 tablets
02:18:25.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.772085 13665 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:18:25.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.941434 13665 rpc_server.cc:307] RPC server started. Bound to: 127.12.220.2:40625
02:18:25.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.941540 13801 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.220.2:40625 every 8 connection(s)
02:18:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.943979 13665 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-1/data/info.pb
02:18:25.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.947368 13168 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu as pid 13665
02:18:25.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:25.947857 13168 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-1/wal/instance
02:18:25.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:25 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185905, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.2@KRBTEST.COM for kudu/127.12.220.60@KRBTEST.COM
02:18:25.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.220.3@KRBTEST.COM; defaulting to no policy
02:18:25.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:25.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.220.3@KRBTEST.COM" created.
02:18:25.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:25 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185905, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.2@KRBTEST.COM for kudu/127.12.220.61@KRBTEST.COM
02:18:26.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:25 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185905, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.2@KRBTEST.COM for kudu/127.12.220.62@KRBTEST.COM
02:18:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:26.008960 13802 heartbeater.cc:344] Connected to a master server at 127.12.220.60:46883
02:18:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:26.009567 13802 heartbeater.cc:461] Registering TS with master...
02:18:26.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:26.011145 13802 heartbeater.cc:507] Master 127.12.220.60:46883 requested a full tablet report, sending...
02:18:26.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:26.013942 13406 ts_manager.cc:194] Registered new tserver with Master: b771fc2735a44c4dbe39fa8aa208e045 (127.12.220.2:40625)
02:18:26.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:26.015666 13406 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.12.220.2@KRBTEST.COM'} at 127.12.220.2:42507
02:18:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:26.019055 13803 heartbeater.cc:344] Connected to a master server at 127.12.220.61:43219
02:18:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:26.019520 13803 heartbeater.cc:461] Registering TS with master...
02:18:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:26.020455 13803 heartbeater.cc:507] Master 127.12.220.61:43219 requested a full tablet report, sending...
02:18:26.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:26.023979 13315 ts_manager.cc:194] Registered new tserver with Master: b771fc2735a44c4dbe39fa8aa208e045 (127.12.220.2:40625)
02:18:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:26.026871 13806 heartbeater.cc:344] Connected to a master server at 127.12.220.62:35095
02:18:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:26.027231 13806 heartbeater.cc:461] Registering TS with master...
02:18:26.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:26.027983 13806 heartbeater.cc:507] Master 127.12.220.62:35095 requested a full tablet report, sending...
02:18:26.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:26.030609 13234 ts_manager.cc:194] Registered new tserver with Master: b771fc2735a44c4dbe39fa8aa208e045 (127.12.220.2:40625)
02:18:26.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:26.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.220.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:26.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.220.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:26.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.12.220.3@KRBTEST.COM; defaulting to no policy
02:18:26.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:26.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.12.220.3@KRBTEST.COM" created.
02:18:26.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:18:26.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.220.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:26.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.220.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab.
02:18:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:26.083048 13168 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:18:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:18:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-2/wal
02:18:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-2/data
02:18:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:18:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:18:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:18:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:18:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:18:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-2/logs
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-2/data/info.pb
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.220.3:0
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.220.3
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.220.3
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.220.62:35095,127.12.220.61:43219,127.12.220.60:46883
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.220.3
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.220.20:39563
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:18:26.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:18:26.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:26.447852 13816 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:18:26.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:26.448419 13816 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:18:26.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:26.449218 13816 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:18:26.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:26.482977 13816 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:18:26.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:26.484033 13816 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.220.3
02:18:26.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:26.524617 13816 tablet_server_runner.cc:78] Tablet server non-default flags:
02:18:26.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:18:26.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.220.20:39563
02:18:26.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:18:26.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:18:26.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-2/data
02:18:26.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-2/wal
02:18:26.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab
02:18:26.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.220.3
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.220.3:0
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-2/data/info.pb
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.220.3
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.220.62:35095,127.12.220.61:43219,127.12.220.60:46883
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.220.3
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-2/logs
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 51f761c37259d8159142e890d4d8106f68bfa520
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 01:43:24 UTC on 5fd53c4cbb9d
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8673
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:18:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:26.526263 13816 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:18:26.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:26.527594 13816 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:18:26.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:26.541663 13822 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:26.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:26 dist-test-slave-df1k krb5kdc[13180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185906, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:18:26.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:26.909443 13383 init.cc:295] Successfully reacquired a new kerberos TGT
02:18:27.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:27.020447 13802 heartbeater.cc:499] Master 127.12.220.60:46883 was elected leader, sending a full tablet report...
02:18:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:26.545526 13825 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:26.544693 13823 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:27.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:27.892637 13824 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1346 milliseconds
02:18:27.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:27.892803 13816 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:18:27.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:27.894361 13816 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:18:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:27.897576 13816 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:18:27.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:27.898939 13816 hybrid_clock.cc:648] HybridClock initialized: now 1761185907898902 us; error 47 us; skew 500 ppm
02:18:27.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:27.899688 13816 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:18:27.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:27 dist-test-slave-df1k krb5kdc[13180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185907, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:18:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:27.908480 13816 init.cc:377] Logged in from keytab as kudu/127.12.220.3@KRBTEST.COM (short username kudu)
02:18:27.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:27.916149 13816 webserver.cc:492] Webserver started at http://127.12.220.3:37261/ using document root <none> and password file <none>
02:18:27.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:27.917090 13816 fs_manager.cc:362] Metadata directory not provided
02:18:27.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:27.917341 13816 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:18:27.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:27.917905 13816 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:18:27.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:27.922433 13816 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-2/data/instance:
02:18:27.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57fccc26cbe5403e9032dbb373fa2574"
02:18:27.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:27 on dist-test-slave-df1k"
02:18:27.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:27.923612 13816 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-2/wal/instance:
02:18:27.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57fccc26cbe5403e9032dbb373fa2574"
02:18:27.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:27 on dist-test-slave-df1k"
02:18:27.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:27.932914 13816 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.007s sys 0.004s
02:18:27.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:27.940030 13837 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:27.941232 13816 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
02:18:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:27.941615 13816 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-2/data,/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-2/wal
02:18:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "57fccc26cbe5403e9032dbb373fa2574"
02:18:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:27 on dist-test-slave-df1k"
02:18:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:27.941968 13816 fs_report.cc:389] FS layout report
02:18:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:18:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-2/wal
02:18:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-2/wal
02:18:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-2/data/data
02:18:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:18:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:18:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:18:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:18:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:18:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:18:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:18:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:18:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:18:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:18:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:18:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:27.999812 13816 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:18:28.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.010703 13816 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:18:28.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.011133 13816 kserver.cc:163] Server-wide thread pool size limit: 3276
02:18:28.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.013926 13816 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:18:28.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.019304 13816 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:18:28.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.019515 13816 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:18:28.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.019758 13816 ts_tablet_manager.cc:616] Registered 0 tablets
02:18:28.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.019905 13816 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:18:28.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.202293 13816 rpc_server.cc:307] RPC server started. Bound to: 127.12.220.3:38805
02:18:28.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.202404 13949 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.220.3:38805 every 8 connection(s)
02:18:28.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.204886 13816 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-2/data/info.pb
02:18:28.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.206301 13168 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu as pid 13816
02:18:28.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.207123 13168 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/ts-2/wal/instance
02:18:28.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:28 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185907, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.3@KRBTEST.COM for kudu/127.12.220.61@KRBTEST.COM
02:18:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:28 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185907, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.3@KRBTEST.COM for kudu/127.12.220.62@KRBTEST.COM
02:18:28.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:28 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185907, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.3@KRBTEST.COM for kudu/127.12.220.60@KRBTEST.COM
02:18:28.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.265887 13952 heartbeater.cc:344] Connected to a master server at 127.12.220.62:35095
02:18:28.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.266429 13952 heartbeater.cc:461] Registering TS with master...
02:18:28.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.267858 13952 heartbeater.cc:507] Master 127.12.220.62:35095 requested a full tablet report, sending...
02:18:28.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.270767 13234 ts_manager.cc:194] Registered new tserver with Master: 57fccc26cbe5403e9032dbb373fa2574 (127.12.220.3:38805)
02:18:28.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.275290 13951 heartbeater.cc:344] Connected to a master server at 127.12.220.61:43219
02:18:28.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.275620 13951 heartbeater.cc:461] Registering TS with master...
02:18:28.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.276540 13951 heartbeater.cc:507] Master 127.12.220.61:43219 requested a full tablet report, sending...
02:18:28.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.280022 13315 ts_manager.cc:194] Registered new tserver with Master: 57fccc26cbe5403e9032dbb373fa2574 (127.12.220.3:38805)
02:18:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.286509 13950 heartbeater.cc:344] Connected to a master server at 127.12.220.60:46883
02:18:28.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.286862 13950 heartbeater.cc:461] Registering TS with master...
02:18:28.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.287843 13950 heartbeater.cc:507] Master 127.12.220.60:46883 requested a full tablet report, sending...
02:18:28.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.290199 13406 ts_manager.cc:194] Registered new tserver with Master: 57fccc26cbe5403e9032dbb373fa2574 (127.12.220.3:38805)
02:18:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.291841 13406 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.12.220.3@KRBTEST.COM'} at 127.12.220.3:46795
02:18:28.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.304752 13168 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
02:18:28.317 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:18:28.318 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
02:18:28.327 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
02:18:28.328 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
02:18:28.331 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.12.220.62"
port: 35095
}
bound_http_address {
host: "127.12.220.62"
port: 45557
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.12.220.61"
port: 43219
}
bound_http_address {
host: "127.12.220.61"
port: 45543
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.12.220.60"
port: 46883
}
bound_http_address {
host: "127.12.220.60"
port: 46715
}
}
}
02:18:28.332 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:18:28.333 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.12.220.1"
port: 42297
}
bound_http_address {
host: "127.12.220.1"
port: 45855
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.12.220.2"
port: 40625
}
bound_http_address {
host: "127.12.220.2"
port: 44627
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.12.220.3"
port: 38805
}
bound_http_address {
host: "127.12.220.3"
port: 37261
}
}
}
02:18:28.333 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:18:28.334 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:18:28.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:28 dist-test-slave-df1k krb5kdc[13180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185908, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:18:28.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:18:28.368 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:18:28.371 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/krb5cc
02:18:28.376 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:18:28.377 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/krb5cc
02:18:28.379 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:18:28.382 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
02:18:28.384 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:18:28.384 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:18:28.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:28 dist-test-slave-df1k krb5kdc[13180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185908, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:18:28.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:18:28.417 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:18:28.417 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/krb5cc
02:18:28.419 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:18:28.419 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.220.62:35095
02:18:28.420 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
02:18:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.421342 13168 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu with pid 13197
02:18:28.453 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:18:28.453 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.220.60:46883
02:18:28.453 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
02:18:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.453675 13168 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu with pid 13360
02:18:28.481 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:18:28.482 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.220.61:43219
02:18:28.482 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
02:18:28.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.482790 13168 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu with pid 13278
02:18:28.516 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:18:28.516 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.220.62:35095
02:18:28.521 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
02:18:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.523589 13168 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:18:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:18:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/wal
02:18:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/data
02:18:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:18:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:18:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:18:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:18:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:18:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:18:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/logs
02:18:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/data/info.pb
02:18:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:18:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:18:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:18:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:18:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:18:28.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.220.62:35095
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.220.62
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45557
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.220.62
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.220.20:39563
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.220.62:35095,127.12.220.61:43219,127.12.220.60:46883
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37799/jwks.json
02:18:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:18:28.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:18:28.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:28.819252 13651 heartbeater.cc:646] Failed to heartbeat to 127.12.220.60:46883 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.12.220.60:46883: connect: Connection refused (error 111)
02:18:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:28.845574 13964 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:18:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:28.846344 13964 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:18:28.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:28.847352 13964 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:18:28.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:28.880394 13964 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:18:28.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:28.880718 13964 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:18:28.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:28.881179 13964 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:18:28.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:28.881553 13964 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:18:28.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:28.881843 13964 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:18:28.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:28.882069 13964 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:18:28.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:28.882225 13964 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37799/jwks.json
02:18:28.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.920378 13964 master_runner.cc:386] Master server non-default flags:
02:18:28.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:18:28.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.220.20:39563
02:18:28.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:18:28.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:18:28.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/data
02:18:28.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/wal
02:18:28.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:18:28.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:18:28.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.220.62:35095,127.12.220.61:43219,127.12.220.60:46883
02:18:28.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab
02:18:28.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.220.62
02:18:28.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:18:28.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:18:28.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:18:28.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.220.62:35095
02:18:28.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:18:28.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:18:28.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:18:28.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37799/jwks.json
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/data/info.pb
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.220.62
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45557
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/logs
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 51f761c37259d8159142e890d4d8106f68bfa520
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 01:43:24 UTC on 5fd53c4cbb9d
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8673
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:18:28.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.921805 13964 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:18:28.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:28.922812 13964 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:18:28.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:28.934363 13972 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:29.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:29.038090 13802 heartbeater.cc:646] Failed to heartbeat to 127.12.220.60:46883 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.12.220.60:46883: connect: Connection refused (error 111)
02:18:29.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:29.296213 13950 heartbeater.cc:499] Master 127.12.220.60:46883 was elected leader, sending a full tablet report...
02:18:29.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:29.303473 13952 heartbeater.cc:646] Failed to heartbeat to 127.12.220.62:35095 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.12.220.62:35095: connect: Connection refused (error 111)
02:18:30.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:28.934922 13973 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:30.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:30.378805 13975 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:30.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:30.382665 13974 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1441 milliseconds
02:18:30.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.382803 13964 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:18:30.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.384330 13964 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:18:30.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.387027 13964 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:18:30.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.389094 13964 hybrid_clock.cc:648] HybridClock initialized: now 1761185910389040 us; error 55 us; skew 500 ppm
02:18:30.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.390120 13964 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:18:30.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:30 dist-test-slave-df1k krb5kdc[13180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185910, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:18:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.396517 13964 init.cc:377] Logged in from keytab as kudu/127.12.220.62@KRBTEST.COM (short username kudu)
02:18:30.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.401957 13964 webserver.cc:492] Webserver started at http://127.12.220.62:45557/ using document root <none> and password file <none>
02:18:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.402909 13964 fs_manager.cc:362] Metadata directory not provided
02:18:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.403118 13964 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:18:30.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.410771 13964 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
02:18:30.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.415385 13994 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:18:30.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.416679 13964 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.004s
02:18:30.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.416996 13964 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/data,/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/wal
02:18:30.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c4b2eb94707a4d91bcf53023db1b2715"
02:18:30.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:12 on dist-test-slave-df1k"
02:18:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.418936 13964 fs_report.cc:389] FS layout report
02:18:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:18:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/wal
02:18:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/wal
02:18:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/data/data
02:18:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:18:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:18:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:18:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:18:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:18:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:18:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:18:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:18:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:18:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:18:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:18:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.478068 13964 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:18:30.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.483943 13964 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:18:30.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.484347 13964 kserver.cc:163] Server-wide thread pool size limit: 3276
02:18:30.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.565181 13964 rpc_server.cc:307] RPC server started. Bound to: 127.12.220.62:35095
02:18:30.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.565311 14045 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.220.62:35095 every 8 connection(s)
02:18:30.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.567859 13964 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-0/data/info.pb
02:18:30.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.570107 13168 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu as pid 13964
02:18:30.571 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:18:30.571 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.220.60:46883
02:18:30.571 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
02:18:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.572741 13168 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:18:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:18:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/wal
02:18:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/data
02:18:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:18:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:18:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:18:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:18:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:18:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:18:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/logs
02:18:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/data/info.pb
02:18:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:18:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:18:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:18:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:18:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:18:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:18:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:18:30.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:18:30.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:18:30.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:18:30.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:18:30.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:18:30.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.220.60:46883
02:18:30.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.220.60
02:18:30.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46715
02:18:30.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab
02:18:30.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.220.60
02:18:30.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:18:30.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:18:30.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:18:30.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:18:30.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.220.20:39563
02:18:30.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:18:30.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:18:30.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:18:30.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:18:30.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.220.62:35095,127.12.220.61:43219,127.12.220.60:46883
02:18:30.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:18:30.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37799/jwks.json
02:18:30.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:18:30.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:18:30.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.580387 14046 sys_catalog.cc:263] Verifying existing consensus state
02:18:30.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.589355 14046 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715: Bootstrap starting.
02:18:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.633971 14046 log.cc:826] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715: Log is configured to *not* fsync() on all Append() calls
02:18:30.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.649684 14046 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:18:30.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.650512 14046 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715: Bootstrap complete.
02:18:30.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.675622 14046 raft_consensus.cc:359] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } }
02:18:30.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.678917 14046 raft_consensus.cc:740] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: c4b2eb94707a4d91bcf53023db1b2715, State: Initialized, Role: FOLLOWER
02:18:30.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.679925 14046 consensus_queue.cc:260] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } }
02:18:30.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.683197 14050 sys_catalog.cc:455] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } } }
02:18:30.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.684080 14050 sys_catalog.cc:458] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [sys.catalog]: This master's current role is: FOLLOWER
02:18:30.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.687951 14046 sys_catalog.cc:565] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [sys.catalog]: configured and running, proceeding with master startup.
02:18:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:30 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185910, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.62@KRBTEST.COM for kudu/127.12.220.62@KRBTEST.COM
02:18:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.729146 14061 catalog_manager.cc:1269] Loaded cluster ID: d3842f9e2fed40dbad6ac30b3c3a89e0
02:18:30.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.729414 14061 catalog_manager.cc:1562] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715: loading cluster ID for follower catalog manager: success
02:18:30.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.737150 14061 catalog_manager.cc:1584] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715: acquiring CA information for follower catalog manager: success
02:18:30.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.742374 14061 catalog_manager.cc:1612] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:18:30.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.750578 13964 master_runner.cc:186] Error getting master registration for 127.12.220.61:43219: Network error: Client connection negotiation failed: client connection to 127.12.220.61:43219: connect: Connection refused (error 111),
02:18:30.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.771344 13964 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:18:30.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.860738 13656 heartbeater.cc:344] Connected to a master server at 127.12.220.62:35095
02:18:30.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.865823 14011 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "9b1513b94be14b8e96e0dbc87c9aada4" instance_seqno: 1761185902596988) as {username='kudu', principal='kudu/127.12.220.1@KRBTEST.COM'} at 127.12.220.1:39503; Asking this server to re-register.
02:18:30.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.867460 13656 heartbeater.cc:461] Registering TS with master...
02:18:30.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:30.870252 14011 ts_manager.cc:194] Registered new tserver with Master: 9b1513b94be14b8e96e0dbc87c9aada4 (127.12.220.1:42297)
02:18:30.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:30.965891 13946 debug-util.cc:398] Leaking SignalData structure 0x7b08000b46c0 after lost signal to thread 13817
02:18:30.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:30.967056 13946 debug-util.cc:398] Leaking SignalData structure 0x7b08000b4620 after lost signal to thread 13949
02:18:31.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:31.010463 14048 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:18:31.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:31.011265 14048 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:18:31.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:31.012392 14048 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:18:31.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:31.071527 14048 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:18:31.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:31.071870 14048 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:18:31.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:31.072332 14048 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:18:31.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:31.072685 14048 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:18:31.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:31.073017 14048 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:18:31.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:31.073369 14048 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:18:31.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:31.073618 14048 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37799/jwks.json
02:18:31.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:31.085637 13806 heartbeater.cc:344] Connected to a master server at 127.12.220.62:35095
02:18:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:31.088488 14011 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "b771fc2735a44c4dbe39fa8aa208e045" instance_seqno: 1761185905900615) as {username='kudu', principal='kudu/127.12.220.2@KRBTEST.COM'} at 127.12.220.2:41393; Asking this server to re-register.
02:18:31.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:31.090037 13806 heartbeater.cc:461] Registering TS with master...
02:18:31.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:31.093091 14011 ts_manager.cc:194] Registered new tserver with Master: b771fc2735a44c4dbe39fa8aa208e045 (127.12.220.2:40625)
02:18:31.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:31.131546 14048 master_runner.cc:386] Master server non-default flags:
02:18:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:18:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.220.20:39563
02:18:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:18:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:18:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/data
02:18:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/wal
02:18:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:18:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:18:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.220.62:35095,127.12.220.61:43219,127.12.220.60:46883
02:18:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab
02:18:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.220.60
02:18:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:18:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:18:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:18:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.220.60:46883
02:18:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:18:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:18:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:18:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37799/jwks.json
02:18:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:18:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:18:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:18:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:18:31.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:18:31.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/data/info.pb
02:18:31.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.220.60
02:18:31.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46715
02:18:31.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:18:31.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:18:31.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:18:31.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:18:31.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:18:31.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/logs
02:18:31.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:18:31.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:18:31.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:18:31.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:18:31.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:18:31.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 51f761c37259d8159142e890d4d8106f68bfa520
02:18:31.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:18:31.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 01:43:24 UTC on 5fd53c4cbb9d
02:18:31.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8673
02:18:31.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:18:31.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:31.133215 14048 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:18:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:31.134661 14048 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:18:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:31.149452 14073 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:31.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:31.327821 13952 heartbeater.cc:344] Connected to a master server at 127.12.220.62:35095
02:18:31.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:31.331684 14011 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "57fccc26cbe5403e9032dbb373fa2574" instance_seqno: 1761185908164278) as {username='kudu', principal='kudu/127.12.220.3@KRBTEST.COM'} at 127.12.220.3:35301; Asking this server to re-register.
02:18:31.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:31.351761 13952 heartbeater.cc:461] Registering TS with master...
02:18:31.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:31.362809 14011 ts_manager.cc:194] Registered new tserver with Master: 57fccc26cbe5403e9032dbb373fa2574 (127.12.220.3:38805)
02:18:31.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:31 dist-test-slave-df1k krb5kdc[13180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185911, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:18:31.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:31.964743 13531 init.cc:295] Successfully reacquired a new kerberos TGT
02:18:32.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:32.366134 14088 raft_consensus.cc:493] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:18:32.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:32.366930 14088 raft_consensus.cc:515] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } }
02:18:32.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:32.391938 13996 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.220.61:43219: connect: Connection refused (error 111)
02:18:32.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:32.393790 14088 leader_election.cc:290] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9f253219e08f4cbea557cc68a0603a85 (127.12.220.61:43219), 423a10751bbd43fda4725dd498e20dda (127.12.220.60:46883)
02:18:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:32.414454 13996 leader_election.cc:336] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9f253219e08f4cbea557cc68a0603a85 (127.12.220.61:43219): Network error: Client connection negotiation failed: client connection to 127.12.220.61:43219: connect: Connection refused (error 111)
02:18:32.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:32.418260 13996 leader_election.cc:336] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 423a10751bbd43fda4725dd498e20dda (127.12.220.60:46883): Network error: Client connection negotiation failed: client connection to 127.12.220.60:46883: connect: Connection refused (error 111)
02:18:32.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:32.418694 13996 leader_election.cc:304] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c4b2eb94707a4d91bcf53023db1b2715; no voters: 423a10751bbd43fda4725dd498e20dda, 9f253219e08f4cbea557cc68a0603a85
02:18:32.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:32.419991 14088 raft_consensus.cc:2749] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
02:18:34.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:34.680169 14105 raft_consensus.cc:493] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:18:34.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:34.681155 14105 raft_consensus.cc:515] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } }
02:18:34.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:34.690879 14105 leader_election.cc:290] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9f253219e08f4cbea557cc68a0603a85 (127.12.220.61:43219), 423a10751bbd43fda4725dd498e20dda (127.12.220.60:46883)
02:18:34.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:34.714541 13996 leader_election.cc:336] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9f253219e08f4cbea557cc68a0603a85 (127.12.220.61:43219): Network error: Client connection negotiation failed: client connection to 127.12.220.61:43219: connect: Connection refused (error 111)
02:18:34.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:34.718849 13996 leader_election.cc:336] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 423a10751bbd43fda4725dd498e20dda (127.12.220.60:46883): Network error: Client connection negotiation failed: client connection to 127.12.220.60:46883: connect: Connection refused (error 111)
02:18:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:34.719729 13996 leader_election.cc:304] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c4b2eb94707a4d91bcf53023db1b2715; no voters: 423a10751bbd43fda4725dd498e20dda, 9f253219e08f4cbea557cc68a0603a85
02:18:34.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:34.721051 14105 raft_consensus.cc:2749] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
02:18:34.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:32.551614 14072 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 14048
02:18:34.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:34.957894 14048 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 3.809s user 1.255s sys 2.114s
02:18:34.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:34.958281 14048 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 3.809s user 1.255s sys 2.114s
02:18:34.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:31.151633 14074 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:34.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:34.959131 14075 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3808 milliseconds
02:18:34.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:34.960378 14076 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:34.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:34.960336 14048 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:18:34.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:34.964821 14048 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:18:34.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:34.967341 14048 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:18:34.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:34.968710 14048 hybrid_clock.cc:648] HybridClock initialized: now 1761185914968662 us; error 62 us; skew 500 ppm
02:18:34.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:34.969599 14048 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:18:34.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:34 dist-test-slave-df1k krb5kdc[13180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185914, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:18:34.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:34.975780 14048 init.cc:377] Logged in from keytab as kudu/127.12.220.60@KRBTEST.COM (short username kudu)
02:18:34.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:34.981757 14048 webserver.cc:492] Webserver started at http://127.12.220.60:46715/ using document root <none> and password file <none>
02:18:34.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:34.982785 14048 fs_manager.cc:362] Metadata directory not provided
02:18:34.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:34.983031 14048 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:18:34.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:34.992687 14048 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.003s sys 0.003s
02:18:34.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:34.998245 14116 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:18:34.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:34.999469 14048 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.005s
02:18:35.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:34.999788 14048 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/data,/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/wal
02:18:35.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "423a10751bbd43fda4725dd498e20dda"
02:18:35.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:16 on dist-test-slave-df1k"
02:18:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.001745 14048 fs_report.cc:389] FS layout report
02:18:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:18:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/wal
02:18:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/wal
02:18:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/data/data
02:18:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:18:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:18:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:18:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:18:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:18:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:18:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:18:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:18:35.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:18:35.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:18:35.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:18:35.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.069228 14048 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:18:35.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.079968 14048 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:18:35.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.080528 14048 kserver.cc:163] Server-wide thread pool size limit: 3276
02:18:35.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.182677 14048 rpc_server.cc:307] RPC server started. Bound to: 127.12.220.60:46883
02:18:35.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.182765 14170 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.220.60:46883 every 8 connection(s)
02:18:35.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.185947 14048 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-2/data/info.pb
02:18:35.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.191608 13168 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu as pid 14048
02:18:35.192 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:18:35.192 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.220.61:43219
02:18:35.193 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
02:18:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.194866 13168 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:18:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu
02:18:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/wal
02:18:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/data
02:18:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:18:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:18:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:18:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:18:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:18:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:18:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/logs
02:18:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/data/info.pb
02:18:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:18:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:18:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:18:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:18:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:18:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:18:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:18:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:18:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:18:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:18:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:18:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:18:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.220.61:43219
02:18:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.220.61
02:18:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45543
02:18:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab
02:18:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.220.61
02:18:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:18:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:18:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:18:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:18:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.220.20:39563
02:18:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:18:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:18:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:18:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:18:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.220.62:35095,127.12.220.61:43219,127.12.220.60:46883
02:18:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:18:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37799/jwks.json
02:18:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:18:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:18:35.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.201079 14171 sys_catalog.cc:263] Verifying existing consensus state
02:18:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.210100 14171 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda: Bootstrap starting.
02:18:35.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.265513 14171 log.cc:826] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda: Log is configured to *not* fsync() on all Append() calls
02:18:35.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:35.278913 14042 debug-util.cc:398] Leaking SignalData structure 0x7b0800086c80 after lost signal to thread 13965
02:18:35.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:35.279608 14042 debug-util.cc:398] Leaking SignalData structure 0x7b0800071740 after lost signal to thread 14045
02:18:35.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.285481 14171 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:18:35.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.286473 14171 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda: Bootstrap complete.
02:18:35.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.320561 14171 raft_consensus.cc:359] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } }
02:18:35.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.324301 14171 raft_consensus.cc:740] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 423a10751bbd43fda4725dd498e20dda, State: Initialized, Role: FOLLOWER
02:18:35.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.325441 14171 consensus_queue.cc:260] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } }
02:18:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.339751 14175 sys_catalog.cc:455] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } } }
02:18:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.341140 14175 sys_catalog.cc:458] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [sys.catalog]: This master's current role is: FOLLOWER
02:18:35.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.343601 14171 sys_catalog.cc:565] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [sys.catalog]: configured and running, proceeding with master startup.
02:18:35.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.444320 14187 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:18:35.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:35 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185914, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.60@KRBTEST.COM for kudu/127.12.220.62@KRBTEST.COM
02:18:35.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.522357 13950 heartbeater.cc:344] Connected to a master server at 127.12.220.60:46883
02:18:35.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.522951 13950 heartbeater.cc:499] Master 127.12.220.60:46883 was elected leader, sending a full tablet report...
02:18:35.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.531116 14133 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "57fccc26cbe5403e9032dbb373fa2574" instance_seqno: 1761185908164278) as {username='kudu', principal='kudu/127.12.220.3@KRBTEST.COM'} at 127.12.220.3:34811; Asking this server to re-register.
02:18:35.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.532871 13950 heartbeater.cc:461] Registering TS with master...
02:18:35.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.538422 14133 ts_manager.cc:194] Registered new tserver with Master: 57fccc26cbe5403e9032dbb373fa2574 (127.12.220.3:38805)
02:18:35.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.537011 14048 master_runner.cc:186] Error getting master registration for 127.12.220.61:43219: Network error: Client connection negotiation failed: client connection to 127.12.220.61:43219: connect: Connection refused (error 111),
02:18:35.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.574873 14048 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:18:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:35 dist-test-slave-df1k krb5kdc[13180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185915, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:18:35.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.680689 13684 init.cc:295] Successfully reacquired a new kerberos TGT
02:18:35.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:35.690829 14173 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:18:35.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:35.691357 14173 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:18:35.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:35.692000 14173 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:18:35.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:35.726624 14173 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:18:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:35.726866 14173 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:18:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:35.727128 14173 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:18:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:35.727330 14173 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:18:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:35.727520 14173 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:18:35.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:35.727722 14173 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:18:35.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:35.727865 14173 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37799/jwks.json
02:18:35.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.767102 14173 master_runner.cc:386] Master server non-default flags:
02:18:35.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:18:35.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.220.20:39563
02:18:35.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:18:35.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:18:35.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/data
02:18:35.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/wal
02:18:35.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:18:35.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:18:35.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.220.62:35095,127.12.220.61:43219,127.12.220.60:46883
02:18:35.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/krb5kdc/kudu.keytab
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.220.61
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.220.61:43219
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37799/jwks.json
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/data/info.pb
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.220.61
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45543
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/logs
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:18:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:18:35.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:18:35.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 51f761c37259d8159142e890d4d8106f68bfa520
02:18:35.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:18:35.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 23 Oct 2025 01:43:24 UTC on 5fd53c4cbb9d
02:18:35.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8673
02:18:35.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:18:35.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.768347 14173 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:18:35.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.769796 14173 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:18:35.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:35.784807 14196 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:35.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.884654 13964 master_runner.cc:186] Error getting master registration for 127.12.220.61:43219: Network error: Client connection negotiation failed: client connection to 127.12.220.61:43219: connect: Connection refused (error 111),
02:18:35.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:35.885313 13964 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:18:35.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:35 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185911, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.1@KRBTEST.COM for kudu/127.12.220.60@KRBTEST.COM
02:18:36.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.018298 13651 heartbeater.cc:344] Connected to a master server at 127.12.220.60:46883
02:18:36.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.022066 14133 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "9b1513b94be14b8e96e0dbc87c9aada4" instance_seqno: 1761185902596988) as {username='kudu', principal='kudu/127.12.220.1@KRBTEST.COM'} at 127.12.220.1:40175; Asking this server to re-register.
02:18:36.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.024394 13651 heartbeater.cc:461] Registering TS with master...
02:18:36.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.028378 14133 ts_manager.cc:194] Registered new tserver with Master: 9b1513b94be14b8e96e0dbc87c9aada4 (127.12.220.1:42297)
02:18:36.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:36 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185915, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.2@KRBTEST.COM for kudu/127.12.220.60@KRBTEST.COM
02:18:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.192164 13802 heartbeater.cc:344] Connected to a master server at 127.12.220.60:46883
02:18:36.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.195477 14133 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "b771fc2735a44c4dbe39fa8aa208e045" instance_seqno: 1761185905900615) as {username='kudu', principal='kudu/127.12.220.2@KRBTEST.COM'} at 127.12.220.2:57459; Asking this server to re-register.
02:18:36.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.197798 13802 heartbeater.cc:461] Registering TS with master...
02:18:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.201270 14133 ts_manager.cc:194] Registered new tserver with Master: b771fc2735a44c4dbe39fa8aa208e045 (127.12.220.2:40625)
02:18:36.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.474320 14187 catalog_manager.cc:1269] Loaded cluster ID: d3842f9e2fed40dbad6ac30b3c3a89e0
02:18:36.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.474686 14187 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda: loading cluster ID for follower catalog manager: success
02:18:36.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.493022 14187 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda: acquiring CA information for follower catalog manager: success
02:18:36.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.518287 14187 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:18:36.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.701272 14211 raft_consensus.cc:493] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:18:36.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.701877 14211 raft_consensus.cc:515] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } }
02:18:36.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.706117 14021 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "423a10751bbd43fda4725dd498e20dda" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "c4b2eb94707a4d91bcf53023db1b2715" is_pre_election: true
02:18:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.706935 14021 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 423a10751bbd43fda4725dd498e20dda in term 1.
02:18:36.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.708249 14118 leader_election.cc:304] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 423a10751bbd43fda4725dd498e20dda, c4b2eb94707a4d91bcf53023db1b2715; no voters:
02:18:36.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.725037 14212 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [term 1 FOLLOWER]: Leader pre-election won for term 2
02:18:36.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.725579 14212 raft_consensus.cc:493] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:18:36.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.726171 14212 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [term 1 FOLLOWER]: Advancing to term 2
02:18:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.739708 14211 leader_election.cc:290] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers c4b2eb94707a4d91bcf53023db1b2715 (127.12.220.62:35095), 9f253219e08f4cbea557cc68a0603a85 (127.12.220.61:43219)
02:18:36.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:36.742909 14118 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.220.61:43219: connect: Connection refused (error 111)
02:18:36.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.766522 14212 raft_consensus.cc:515] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } }
02:18:36.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:36.769510 14118 leader_election.cc:336] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9f253219e08f4cbea557cc68a0603a85 (127.12.220.61:43219): Network error: Client connection negotiation failed: client connection to 127.12.220.61:43219: connect: Connection refused (error 111)
02:18:36.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.772158 14021 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "423a10751bbd43fda4725dd498e20dda" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "c4b2eb94707a4d91bcf53023db1b2715"
02:18:36.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.772996 14021 raft_consensus.cc:3060] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [term 1 FOLLOWER]: Advancing to term 2
02:18:36.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.783180 14021 raft_consensus.cc:2468] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 423a10751bbd43fda4725dd498e20dda in term 2.
02:18:36.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.784703 14118 leader_election.cc:304] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 423a10751bbd43fda4725dd498e20dda, c4b2eb94707a4d91bcf53023db1b2715; no voters:
02:18:36.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.785648 14211 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [term 2 FOLLOWER]: Leader election won for term 2
02:18:36.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.787835 14211 raft_consensus.cc:697] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [term 2 LEADER]: Becoming Leader. State: Replica: 423a10751bbd43fda4725dd498e20dda, State: Running, Role: LEADER
02:18:36.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.793774 14211 consensus_queue.cc:237] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } }
02:18:36.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.816303 14212 leader_election.cc:290] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [CANDIDATE]: Term 2 election: Requested vote from peers c4b2eb94707a4d91bcf53023db1b2715 (127.12.220.62:35095), 9f253219e08f4cbea557cc68a0603a85 (127.12.220.61:43219)
02:18:36.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.819926 14217 sys_catalog.cc:455] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [sys.catalog]: SysCatalogTable state changed. Reason: New leader 423a10751bbd43fda4725dd498e20dda. Latest consensus state: current_term: 2 leader_uuid: "423a10751bbd43fda4725dd498e20dda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } } }
02:18:36.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:36.820520 14118 leader_election.cc:336] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 9f253219e08f4cbea557cc68a0603a85 (127.12.220.61:43219): Network error: Client connection negotiation failed: client connection to 127.12.220.61:43219: connect: Connection refused (error 111)
02:18:36.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.820811 14217 sys_catalog.cc:458] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [sys.catalog]: This master's current role is: LEADER
02:18:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.836001 14219 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
02:18:36.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.852241 14219 catalog_manager.cc:1494] Initializing Kudu cluster ID...
02:18:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.864110 14219 catalog_manager.cc:1269] Loaded cluster ID: d3842f9e2fed40dbad6ac30b3c3a89e0
02:18:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.864434 14219 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
02:18:36.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.871294 14219 catalog_manager.cc:1514] Loading token signing keys...
02:18:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.883697 14219 catalog_manager.cc:6033] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda: Loaded TSK: 0
02:18:36.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:36.888625 14219 catalog_manager.cc:1524] Initializing in-progress tserver states...
02:18:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:37.210673 14118 consensus_peers.cc:597] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda -> Peer 9f253219e08f4cbea557cc68a0603a85 (127.12.220.61:43219): Couldn't send request to peer 9f253219e08f4cbea557cc68a0603a85. Status: Network error: Client connection negotiation failed: client connection to 127.12.220.61:43219: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:18:37.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:37.357519 14021 raft_consensus.cc:1275] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [term 2 FOLLOWER]: Refusing update from remote peer 423a10751bbd43fda4725dd498e20dda: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
02:18:37.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:37.360821 14217 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [LEADER]: Connected to new peer: Peer: permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.003s
02:18:37.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:37.427737 14222 sys_catalog.cc:455] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 423a10751bbd43fda4725dd498e20dda. Latest consensus state: current_term: 2 leader_uuid: "423a10751bbd43fda4725dd498e20dda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } } }
02:18:37.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:37.428507 14222 sys_catalog.cc:458] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [sys.catalog]: This master's current role is: FOLLOWER
02:18:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:37.442334 14222 sys_catalog.cc:455] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "423a10751bbd43fda4725dd498e20dda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } } }
02:18:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:37.443094 14222 sys_catalog.cc:458] T 00000000000000000000000000000000 P c4b2eb94707a4d91bcf53023db1b2715 [sys.catalog]: This master's current role is: FOLLOWER
02:18:37.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:37.450793 14225 sys_catalog.cc:455] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "423a10751bbd43fda4725dd498e20dda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } } }
02:18:37.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:37.452906 14225 sys_catalog.cc:458] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [sys.catalog]: This master's current role is: LEADER
02:18:37.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:37 dist-test-slave-df1k krb5kdc[13180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185917, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:18:37.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:37.917017 13833 init.cc:295] Successfully reacquired a new kerberos TGT
02:18:38.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.035655 13651 heartbeater.cc:499] Master 127.12.220.60:46883 was elected leader, sending a full tablet report...
02:18:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.207468 13802 heartbeater.cc:499] Master 127.12.220.60:46883 was elected leader, sending a full tablet report...
02:18:38.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:35.789951 14197 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:38.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:38.524910 14173 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.736s user 0.879s sys 1.394s
02:18:38.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:37.184788 14195 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 14173
02:18:38.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:38.530339 14173 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.741s user 0.879s sys 1.394s
02:18:38.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:38.527702 14199 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:18:38.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:38.534202 14198 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2731 milliseconds
02:18:38.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.534215 14173 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:18:38.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.535805 14173 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:18:38.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.550771 14173 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:18:38.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.551086 13950 heartbeater.cc:499] Master 127.12.220.60:46883 was elected leader, sending a full tablet report...
02:18:38.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.554478 14173 hybrid_clock.cc:648] HybridClock initialized: now 1761185918554417 us; error 50 us; skew 500 ppm
02:18:38.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.555680 14173 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:18:38.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:38 dist-test-slave-df1k krb5kdc[13180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185918, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:18:38.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.563935 14173 init.cc:377] Logged in from keytab as kudu/127.12.220.61@KRBTEST.COM (short username kudu)
02:18:38.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.570348 14173 webserver.cc:492] Webserver started at http://127.12.220.61:45543/ using document root <none> and password file <none>
02:18:38.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.571782 14173 fs_manager.cc:362] Metadata directory not provided
02:18:38.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.572211 14173 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:18:38.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.583552 14173 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.005s sys 0.004s
02:18:38.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.591111 14241 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:18:38.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.592851 14173 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.002s sys 0.003s
02:18:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.593446 14173 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/data,/tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/wal
02:18:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f253219e08f4cbea557cc68a0603a85"
02:18:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-23 02:18:14 on dist-test-slave-df1k"
02:18:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.596349 14173 fs_report.cc:389] FS layout report
02:18:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:18:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/wal
02:18:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/wal
02:18:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/data/data
02:18:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:18:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:18:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:18:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:18:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:18:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:18:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:18:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:18:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:18:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:18:38.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:18:38.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.670142 14173 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:18:38.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.677979 14173 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:18:38.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.678829 14173 kserver.cc:163] Server-wide thread pool size limit: 3276
02:18:38.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.794569 14173 rpc_server.cc:307] RPC server started. Bound to: 127.12.220.61:43219
02:18:38.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.794598 14292 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.220.61:43219 every 8 connection(s)
02:18:38.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.799189 14173 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskF8zSoX/test-tmp/mini-kudu-cluster1392258908257994464/master-1/data/info.pb
02:18:38.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.805130 13168 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu as pid 14173
02:18:38.807 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:18:38.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.818919 14293 sys_catalog.cc:263] Verifying existing consensus state
02:18:38.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.828755 14293 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85: Bootstrap starting.
02:18:38.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.887636 14293 log.cc:826] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85: Log is configured to *not* fsync() on all Append() calls
02:18:38.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.917128 14293 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:18:38.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.918365 14293 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85: Bootstrap complete.
02:18:38.958 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.220.62:35095(127.12.220.62:35095)] connecting to peer
02:18:38.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.969105 14293 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } }
02:18:38.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.972663 14293 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9f253219e08f4cbea557cc68a0603a85, State: Initialized, Role: FOLLOWER
02:18:38.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.973852 14293 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } }
02:18:38.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:38.995498 14293 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [sys.catalog]: configured and running, proceeding with master startup.
02:18:39.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:39.006037 14298 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } } }
02:18:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:39.016357 14298 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [sys.catalog]: This master's current role is: FOLLOWER
02:18:39.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:39 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185911, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.1@KRBTEST.COM for kudu/127.12.220.61@KRBTEST.COM
02:18:39.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:39.068343 13652 heartbeater.cc:344] Connected to a master server at 127.12.220.61:43219
02:18:39.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:39 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185914, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.60@KRBTEST.COM for kudu/127.12.220.61@KRBTEST.COM
02:18:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:39.168690 14268 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [term 1 FOLLOWER]: Advancing to term 2
02:18:39.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:39.169910 14313 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:18:39.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:39 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185918, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.61@KRBTEST.COM for kudu/127.12.220.62@KRBTEST.COM
02:18:39.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:39.215129 14298 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 423a10751bbd43fda4725dd498e20dda. Latest consensus state: current_term: 2 leader_uuid: "423a10751bbd43fda4725dd498e20dda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } } }
02:18:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:39.215852 14298 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [sys.catalog]: This master's current role is: FOLLOWER
02:18:39.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:39.217995 14318 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "423a10751bbd43fda4725dd498e20dda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } } }
02:18:39.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:39.218715 14318 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f253219e08f4cbea557cc68a0603a85 [sys.catalog]: This master's current role is: FOLLOWER
02:18:39.242 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.220.60:46883(127.12.220.60:46883)] connecting to peer
02:18:39.248 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.220.61:43219(127.12.220.61:43219)] connecting to peer
02:18:39.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:39 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185915, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.2@KRBTEST.COM for kudu/127.12.220.61@KRBTEST.COM
02:18:39.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:39 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185918, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.61@KRBTEST.COM for kudu/127.12.220.61@KRBTEST.COM
02:18:39.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:39.287401 13803 heartbeater.cc:344] Connected to a master server at 127.12.220.61:43219
02:18:39.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:39.293987 14258 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "b771fc2735a44c4dbe39fa8aa208e045" instance_seqno: 1761185905900615) as {username='kudu', principal='kudu/127.12.220.2@KRBTEST.COM'} at 127.12.220.2:47625; Asking this server to re-register.
02:18:39.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:39.295753 13803 heartbeater.cc:461] Registering TS with master...
02:18:39.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:39.299180 14257 ts_manager.cc:194] Registered new tserver with Master: b771fc2735a44c4dbe39fa8aa208e045 (127.12.220.2:40625)
02:18:39.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:39 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185918, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.61@KRBTEST.COM for kudu/127.12.220.60@KRBTEST.COM
02:18:39.332 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.12.220.62:35095(127.12.220.62:35095)] Successfully connected to peer
02:18:39.336 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.12.220.60:46883(127.12.220.60:46883)] Successfully connected to peer
02:18:39.426 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.12.220.61:43219(127.12.220.61:43219)] Successfully connected to peer
02:18:39.447 [WARN - kudu-nio-1] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
02:18:39.450 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.220.60:46883(127.12.220.60:46883)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
02:18:39.451 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.220.62:35095(127.12.220.62:35095)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
02:18:39.460 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.220.62:35095: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
02:18:39.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:39.463182 14316 negotiation.cc:337] Failed RPC negotiation. Trace:
02:18:39.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.326944 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:52432 (local address 127.12.220.62:35095)
02:18:39.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.328152 (+ 1208us) server_negotiation.cc:207] Beginning negotiation
02:18:39.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.328170 (+ 18us) server_negotiation.cc:400] Waiting for connection header
02:18:39.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.395583 (+ 67413us) server_negotiation.cc:408] Connection header received
02:18:39.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.427562 (+ 31979us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
02:18:39.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.427608 (+ 46us) server_negotiation.cc:462] Received NEGOTIATE request from client
02:18:39.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.427825 (+ 217us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
02:18:39.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.427999 (+ 174us) server_negotiation.cc:227] Negotiated authn=SASL
02:18:39.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.461804 (+ 33805us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:52432: BlockingRecv error: recv got EOF from 127.0.0.1:52432 (error 108)
02:18:39.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":790}
02:18:39.464 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.220.60:46883: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
02:18:39.466 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.220.61:43219(127.12.220.61:43219)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
02:18:39.466 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.220.61:43219: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
02:18:39.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:39.466799 14323 negotiation.cc:337] Failed RPC negotiation. Trace:
02:18:39.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.328809 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:46152 (local address 127.12.220.60:46883)
02:18:39.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.340030 (+ 11221us) server_negotiation.cc:207] Beginning negotiation
02:18:39.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.340055 (+ 25us) server_negotiation.cc:400] Waiting for connection header
02:18:39.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.395473 (+ 55418us) server_negotiation.cc:408] Connection header received
02:18:39.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.426667 (+ 31194us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
02:18:39.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.426698 (+ 31us) server_negotiation.cc:462] Received NEGOTIATE request from client
02:18:39.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.426952 (+ 254us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
02:18:39.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.427167 (+ 215us) server_negotiation.cc:227] Negotiated authn=SASL
02:18:39.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.465288 (+ 38121us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:46152: BlockingRecv error: recv got EOF from 127.0.0.1:46152 (error 108)
02:18:39.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":10639,"thread_start_us":11616,"threads_started":1}
02:18:39.468 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.12.220.60:46883(127.12.220.60:46883)] handling channelInactive
02:18:39.473 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=661), Traces: [0ms] refreshing cache from master, [92ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.220.62:35095, [419ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.220.60:46883, [423ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.220.61:43219, [628ms] Sub RPC ConnectToMaster: received response from server master-127.12.220.60:46883: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [629ms] Sub RPC ConnectToMaster: received response from server master-127.12.220.62:35095: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [643ms] Sub RPC ConnectToMaster: received response from server master-127.12.220.61:43219: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.12.220.62:35095,127.12.220.60:46883,127.12.220.61:43219). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) [main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) [main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) [async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) [main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) [main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) [main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) [netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.12.220.62:35095,127.12.220.60:46883,127.12.220.61:43219). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
02:18:39.479 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=668), Traces: [0ms] refreshing cache from master, [92ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.220.62:35095, [419ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.220.60:46883, [423ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.220.61:43219, [628ms] Sub RPC ConnectToMaster: received response from server master-127.12.220.60:46883: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [629ms] Sub RPC ConnectToMaster: received response from server master-127.12.220.62:35095: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [643ms] Sub RPC ConnectToMaster: received response from server master-127.12.220.61:43219: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) with no deferred attached!
02:18:39.482 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.12.220.62:35095(127.12.220.62:35095)] handling channelInactive
02:18:39.482 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.12.220.61:43219(127.12.220.61:43219)] handling channelInactive
02:18:39.482 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.12.220.60:46883(127.12.220.60:46883)] disconnecting while in state TERMINATED
02:18:39.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:39.483793 14322 negotiation.cc:337] Failed RPC negotiation. Trace:
02:18:39.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.330338 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:57962 (local address 127.12.220.61:43219)
02:18:39.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.331659 (+ 1321us) server_negotiation.cc:207] Beginning negotiation
02:18:39.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.331693 (+ 34us) server_negotiation.cc:400] Waiting for connection header
02:18:39.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.427366 (+ 95673us) server_negotiation.cc:408] Connection header received
02:18:39.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.428194 (+ 828us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
02:18:39.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.428225 (+ 31us) server_negotiation.cc:462] Received NEGOTIATE request from client
02:18:39.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.428463 (+ 238us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
02:18:39.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.428616 (+ 153us) server_negotiation.cc:227] Negotiated authn=SASL
02:18:39.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1023 02:18:39.482148 (+ 53532us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:57962: BlockingRecv error: recv got EOF from 127.0.0.1:57962 (error 108)
02:18:39.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":367}
02:18:39.486 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.12.220.62:35095(127.12.220.62:35095)] disconnecting while in state TERMINATED
02:18:39.486 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.12.220.61:43219(127.12.220.61:43219)] disconnecting while in state TERMINATED
02:18:39.494 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:18:39.498 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:18:39.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:39.522197 13168 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu with pid 13515
02:18:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:39.571367 13168 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu with pid 13665
02:18:39.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 23 02:18:39 dist-test-slave-df1k krb5kdc[13180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761185917, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.220.3@KRBTEST.COM for kudu/127.12.220.61@KRBTEST.COM
02:18:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:39.591924 14225 sys_catalog.cc:455] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "423a10751bbd43fda4725dd498e20dda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4b2eb94707a4d91bcf53023db1b2715" member_type: VOTER last_known_addr { host: "127.12.220.62" port: 35095 } } peers { permanent_uuid: "9f253219e08f4cbea557cc68a0603a85" member_type: VOTER last_known_addr { host: "127.12.220.61" port: 43219 } } peers { permanent_uuid: "423a10751bbd43fda4725dd498e20dda" member_type: VOTER last_known_addr { host: "127.12.220.60" port: 46883 } } }
02:18:39.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:39.592864 14225 sys_catalog.cc:458] T 00000000000000000000000000000000 P 423a10751bbd43fda4725dd498e20dda [sys.catalog]: This master's current role is: LEADER
02:18:39.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:39.605774 13168 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu with pid 13816
02:18:39.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251023 02:18:39.639694 14245 connection.cc:537] server connection from 127.12.220.3:40399 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
02:18:39.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:39.646060 13168 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu with pid 13964
02:18:39.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:39.673691 13168 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu with pid 14173
02:18:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:39.698403 13168 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu with pid 14048
02:18:40.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-23T02:18:40Z chronyd exiting
02:18:41.950 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:18:41.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:41.951342 12439 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu with pid 12741
02:18:41.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:41.977344 12439 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu with pid 12879
02:18:42.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:42.004949 12439 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu with pid 13021
02:18:42.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:42.030383 12439 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu with pid 12451
02:18:42.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:42.060292 12439 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu with pid 12521
02:18:42.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251023 02:18:42.088745 12439 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskF8zSoX/build/tsan/bin/kudu with pid 12596
02:18:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-23T02:18:43Z chronyd exiting
02:18:44.267 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.12.220.62:35095,127.12.220.60:46883,127.12.220.61:43219). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.12.220.62:35095,127.12.220.60:46883,127.12.220.61:43219). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
02:18:44.669 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
02:18:44.669 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.12.220.62:35095,127.12.220.60:46883,127.12.220.61:43219). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.12.220.62:35095,127.12.220.60:46883,127.12.220.61:43219). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more