01:31:52.096 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
01:31:52.108 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskCdr2xp/test-tmp
01:31:52.111 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
01:31:52.114 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskCdr2xp/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
01:31:52.382 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
01:31:52.535 [DEBUG - main] (MiniKuduCluster.java:195) Response:
01:31:52.536 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
01:31:52.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:52.540063 374 mini_oidc.cc:152] Starting JWKS server
01:31:52.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:52.544643 374 webserver.cc:460] Webserver started at http://127.0.0.1:38477/ using document root <none> and password file <none>
01:31:52.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:52.545218 374 mini_oidc.cc:189] Starting OIDC Discovery server
01:31:52.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:52.547245 374 webserver.cc:460] Webserver started at http://127.0.0.1:35921/ using document root <none> and password file <none>
01:31:52.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T01:31:52Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
01:31:52.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T01:31:52Z Disabled control of system clock
01:31:52.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:52.600744 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:31:52.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:31:52.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-0/wal
01:31:52.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-0/data
01:31:52.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:31:52.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:31:52.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:31:52.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:31:52.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:31:52.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:31:52.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-0/logs
01:31:52.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-0/data/info.pb
01:31:52.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:31:52.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:31:52.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:31:52.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:31:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:31:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:31:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:31:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:31:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:31:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:31:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:31:52.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:31:52.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:39385
01:31:52.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
01:31:52.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:31:52.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:38785
01:31:52.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:31:52.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:31:52.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:31:52.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:31:52.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:39385,127.0.93.189:36365,127.0.93.188:38631
01:31:52.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:31:52.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38477/jwks.json with env {}
01:31:52.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:52.945327 386 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:31:52.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:52.946125 386 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:31:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:52.946875 386 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:31:52.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:52.980357 386 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:31:52.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:52.980660 386 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:31:52.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:52.980845 386 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:31:52.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:52.981033 386 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:31:52.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:52.981210 386 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:31:52.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:52.981340 386 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38477/jwks.json
01:31:53.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:53.019891 386 master_runner.cc:386] Master server non-default flags:
01:31:53.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:31:53.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:38785
01:31:53.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:31:53.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:31:53.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-0/data
01:31:53.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-0/wal
01:31:53.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:31:53.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:39385,127.0.93.189:36365,127.0.93.188:38631
01:31:53.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:31:53.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:31:53.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:31:53.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:39385
01:31:53.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:31:53.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:31:53.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:31:53.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38477/jwks.json
01:31:53.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:31:53.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:31:53.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-0/data/info.pb
01:31:53.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
01:31:53.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:31:53.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:31:53.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:31:53.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:31:53.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:31:53.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:31:53.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-0/logs
01:31:53.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:31:53.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:31:53.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:31:53.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:31:53.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:31:53.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:31:53.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:31:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:31:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:31:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:31:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:53.021801 386 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:31:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:53.024349 386 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:31:53.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:53.049027 395 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:31:54.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:53.049054 392 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:31:54.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:53.049053 393 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:31:54.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:53.050946 386 server_base.cc:1034] running on GCE node
01:31:54.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.273797 386 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:31:54.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.276520 386 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:31:54.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.277907 386 hybrid_clock.cc:648] HybridClock initialized: now 1790645514277855 us; error 63 us; skew 500 ppm
01:31:54.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.278728 386 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:31:54.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.286511 386 webserver.cc:460] Webserver started at http://127.0.93.190:39651/ using document root <none> and password file <none>
01:31:54.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.287424 386 fs_manager.cc:359] Metadata directory not provided
01:31:54.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.287698 386 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:31:54.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.288100 386 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:31:54.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.293684 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-0/data/instance:
01:31:54.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "58b9e5311d8a4d10b289ae758bf1cb8e"
01:31:54.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:31:54 on dist-test-slave-vbz4"
01:31:54.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.295495 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-0/wal/instance:
01:31:54.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "58b9e5311d8a4d10b289ae758bf1cb8e"
01:31:54.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:31:54 on dist-test-slave-vbz4"
01:31:54.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.303551 386 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
01:31:54.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.309176 402 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:31:54.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.310858 386 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
01:31:54.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.311252 386 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-0/data,/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-0/wal
01:31:54.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "58b9e5311d8a4d10b289ae758bf1cb8e"
01:31:54.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:31:54 on dist-test-slave-vbz4"
01:31:54.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.311764 386 fs_report.cc:383] FS layout report
01:31:54.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:31:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-0/wal
01:31:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-0/wal
01:31:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-0/data/data
01:31:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:31:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:31:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:31:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:31:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:31:54.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:31:54.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:31:54.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:31:54.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:31:54.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:31:54.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:31:54.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.354578 386 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:31:54.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.356319 386 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:31:54.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.356881 386 kserver.cc:163] Server-wide thread pool size limit: 3276
01:31:54.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.440184 386 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.190:39385
01:31:54.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.440297 453 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.190:39385 every 8 connection(s)
01:31:54.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.443428 386 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-0/data/info.pb
01:31:54.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.447623 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu as pid 386
01:31:54.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.448201 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-0/wal/instance
01:31:54.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.450469 454 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:31:54.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.452160 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:31:54.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:31:54.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-1/wal
01:31:54.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-1/data
01:31:54.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:31:54.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:31:54.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:31:54.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:31:54.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:31:54.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:31:54.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-1/logs
01:31:54.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-1/data/info.pb
01:31:54.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:31:54.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:31:54.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:31:54.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:31:54.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:31:54.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:31:54.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:31:54.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:31:54.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:31:54.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:31:54.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:31:54.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:31:54.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:36365
01:31:54.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
01:31:54.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:31:54.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:38785
01:31:54.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:31:54.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:31:54.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:31:54.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:31:54.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:39385,127.0.93.189:36365,127.0.93.188:38631
01:31:54.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:31:54.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38477/jwks.json with env {}
01:31:54.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.471288 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 39385 } has no permanent_uuid. Determining permanent_uuid...
01:31:54.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.504675 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } has no permanent_uuid. Determining permanent_uuid...
01:31:54.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:54.507544 405 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.189:36365: connect: Connection refused (error 111)
01:31:54.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:54.512121 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:36365: Network error: Client connection negotiation failed: client connection to 127.0.93.189:36365: connect: Connection refused (error 111)
01:31:54.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.561683 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } attempt: 1
01:31:54.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:54.566082 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:36365: Network error: Client connection negotiation failed: client connection to 127.0.93.189:36365: connect: Connection refused (error 111)
01:31:54.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.634550 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } attempt: 2
01:31:54.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:54.638818 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:36365: Network error: Client connection negotiation failed: client connection to 127.0.93.189:36365: connect: Connection refused (error 111)
01:31:54.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.730469 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } attempt: 3
01:31:54.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:54.734738 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:36365: Network error: Client connection negotiation failed: client connection to 127.0.93.189:36365: connect: Connection refused (error 111)
01:31:54.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:54.798117 456 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:31:54.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:54.798864 456 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:31:54.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:54.799482 456 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:31:54.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:54.833082 456 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:31:54.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:54.833388 456 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:31:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:54.833587 456 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:31:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:54.833871 456 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:31:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:54.834183 456 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:31:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:54.834580 456 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38477/jwks.json
01:31:54.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.872399 456 master_runner.cc:386] Master server non-default flags:
01:31:54.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:31:54.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:38785
01:31:54.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:31:54.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:31:54.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-1/data
01:31:54.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-1/wal
01:31:54.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:31:54.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:39385,127.0.93.189:36365,127.0.93.188:38631
01:31:54.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:31:54.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:31:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:31:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:36365
01:31:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:31:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:31:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:31:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38477/jwks.json
01:31:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:31:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:31:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-1/data/info.pb
01:31:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
01:31:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:31:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:31:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:31:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:31:54.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:31:54.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:31:54.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-1/logs
01:31:54.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:31:54.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:31:54.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:31:54.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:31:54.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:31:54.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:31:54.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:31:54.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:31:54.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:31:54.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:31:54.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.873677 456 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:31:54.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.875888 456 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:31:54.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.878599 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } attempt: 4
01:31:54.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:54.883009 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:36365: Network error: Client connection negotiation failed: client connection to 127.0.93.189:36365: connect: Connection refused (error 111)
01:31:55.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:55.182471 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } attempt: 5
01:31:55.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:55.187659 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:36365: Network error: Client connection negotiation failed: client connection to 127.0.93.189:36365: connect: Connection refused (error 111)
01:31:55.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:55.735055 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } attempt: 6
01:31:55.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:55.747387 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:36365: Network error: Client connection negotiation failed: client connection to 127.0.93.189:36365: connect: Connection refused (error 111)
01:31:56.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:54.902890 456 server_base.cc:1034] running on GCE node
01:31:56.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:54.899345 465 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:31:56.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:54.899360 468 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:31:56.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:54.899391 466 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:31:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.150460 456 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:31:56.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.154139 456 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:31:56.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.155618 456 hybrid_clock.cc:648] HybridClock initialized: now 1790645516155543 us; error 92 us; skew 500 ppm
01:31:56.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.156888 456 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:31:56.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.163844 456 webserver.cc:460] Webserver started at http://127.0.93.189:45489/ using document root <none> and password file <none>
01:31:56.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.164848 456 fs_manager.cc:359] Metadata directory not provided
01:31:56.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.165045 456 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:31:56.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.165431 456 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:31:56.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.171008 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-1/data/instance:
01:31:56.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b652e14749ba4be2ba681f496c77c6d2"
01:31:56.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:31:56 on dist-test-slave-vbz4"
01:31:56.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.172385 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-1/wal/instance:
01:31:56.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b652e14749ba4be2ba681f496c77c6d2"
01:31:56.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:31:56 on dist-test-slave-vbz4"
01:31:56.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.179966 456 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
01:31:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.186719 477 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:31:56.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.188043 456 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.004s
01:31:56.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.188444 456 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-1/data,/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-1/wal
01:31:56.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b652e14749ba4be2ba681f496c77c6d2"
01:31:56.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:31:56 on dist-test-slave-vbz4"
01:31:56.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.188882 456 fs_report.cc:383] FS layout report
01:31:56.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:31:56.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-1/wal
01:31:56.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-1/wal
01:31:56.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-1/data/data
01:31:56.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:31:56.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:31:56.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:31:56.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:31:56.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:31:56.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:31:56.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:31:56.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:31:56.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:31:56.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:31:56.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:31:56.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.275394 456 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:31:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.277004 456 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:31:56.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.277428 456 kserver.cc:163] Server-wide thread pool size limit: 3276
01:31:56.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.361742 456 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.189:36365
01:31:56.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.361827 528 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.189:36365 every 8 connection(s)
01:31:56.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.364812 456 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-1/data/info.pb
01:31:56.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.368661 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu as pid 456
01:31:56.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.369200 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-1/wal/instance
01:31:56.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.371990 529 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:31:56.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.372249 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:31:56.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:31:56.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-2/wal
01:31:56.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-2/data
01:31:56.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:31:56.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:31:56.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:31:56.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:31:56.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:31:56.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:31:56.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-2/logs
01:31:56.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-2/data/info.pb
01:31:56.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:31:56.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:31:56.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:31:56.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:31:56.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:31:56.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:31:56.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:31:56.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:31:56.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:31:56.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:31:56.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:31:56.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:31:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:38631
01:31:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
01:31:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:31:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:38785
01:31:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:31:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:31:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:31:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:31:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:39385,127.0.93.189:36365,127.0.93.188:38631
01:31:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:31:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38477/jwks.json with env {}
01:31:56.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.388989 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 39385 } has no permanent_uuid. Determining permanent_uuid...
01:31:56.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.418757 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } has no permanent_uuid. Determining permanent_uuid...
01:31:56.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.437394 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } has no permanent_uuid. Determining permanent_uuid...
01:31:56.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:56.439824 480 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38631: connect: Connection refused (error 111)
01:31:56.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:56.449582 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38631: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38631: connect: Connection refused (error 111)
01:31:56.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.499195 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } attempt: 1
01:31:56.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:56.503819 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38631: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38631: connect: Connection refused (error 111)
01:31:56.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.572234 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } attempt: 2
01:31:56.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:56.576920 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38631: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38631: connect: Connection refused (error 111)
01:31:56.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.668361 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } attempt: 3
01:31:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:56.672199 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38631: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38631: connect: Connection refused (error 111)
01:31:56.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:56.730414 531 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:31:56.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:56.731168 531 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:31:56.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:56.731896 531 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:31:56.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:56.765005 531 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:31:56.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:56.765427 531 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:31:56.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:56.765764 531 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:31:56.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:56.766134 531 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:31:56.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:56.766465 531 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:31:56.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:56.766707 531 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38477/jwks.json
01:31:56.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.802433 531 master_runner.cc:386] Master server non-default flags:
01:31:56.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:31:56.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:38785
01:31:56.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:31:56.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:31:56.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-2/data
01:31:56.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-2/wal
01:31:56.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:31:56.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:39385,127.0.93.189:36365,127.0.93.188:38631
01:31:56.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:31:56.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:31:56.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:31:56.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:38631
01:31:56.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:31:56.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:31:56.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:31:56.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38477/jwks.json
01:31:56.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:31:56.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:31:56.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-2/data/info.pb
01:31:56.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
01:31:56.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:31:56.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:31:56.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:31:56.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:31:56.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:31:56.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:31:56.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-2/logs
01:31:56.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:31:56.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:31:56.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:31:56.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:31:56.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:31:56.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:31:56.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:31:56.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:31:56.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:31:56.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:31:56.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.804083 531 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:31:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.806239 531 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:31:56.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.808152 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } attempt: 7
01:31:56.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.815665 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } attempt: 4
01:31:56.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:56.821111 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38631: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38631: connect: Connection refused (error 111)
01:31:56.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.826454 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } has no permanent_uuid. Determining permanent_uuid...
01:31:56.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:56.833067 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38631: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38631: connect: Connection refused (error 111)
01:31:56.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:56.836737 545 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:31:56.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.891535 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } attempt: 1
01:31:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:56.895748 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38631: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38631: connect: Connection refused (error 111)
01:31:56.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.977386 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } attempt: 2
01:31:56.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:56.981953 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38631: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38631: connect: Connection refused (error 111)
01:31:57.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:57.067631 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } attempt: 3
01:31:57.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:57.073362 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38631: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38631: connect: Connection refused (error 111)
01:31:57.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:57.120599 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } attempt: 5
01:31:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:57.126392 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38631: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38631: connect: Connection refused (error 111)
01:31:57.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:57.213827 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } attempt: 4
01:31:57.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:57.218341 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38631: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38631: connect: Connection refused (error 111)
01:31:57.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:57.501981 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } attempt: 5
01:31:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:57.506901 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38631: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38631: connect: Connection refused (error 111)
01:31:57.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:57.673946 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } attempt: 6
01:31:57.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:57.683954 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38631: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38631: connect: Connection refused (error 111)
01:31:58.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.059541 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } attempt: 6
01:31:58.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:58.066514 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38631: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38631: connect: Connection refused (error 111)
01:31:58.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:56.837621 542 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:31:58.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:56.838340 531 server_base.cc:1034] running on GCE node
01:31:58.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:56.838442 543 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:31:58.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.101888 531 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:31:58.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.104954 531 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:31:58.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.106735 531 hybrid_clock.cc:648] HybridClock initialized: now 1790645518106688 us; error 59 us; skew 500 ppm
01:31:58.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.107643 531 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:31:58.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.121358 531 webserver.cc:460] Webserver started at http://127.0.93.188:37149/ using document root <none> and password file <none>
01:31:58.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.122381 531 fs_manager.cc:359] Metadata directory not provided
01:31:58.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.122563 531 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:31:58.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.122951 531 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:31:58.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.128100 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-2/data/instance:
01:31:58.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef"
01:31:58.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:31:58 on dist-test-slave-vbz4"
01:31:58.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.129251 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-2/wal/instance:
01:31:58.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef"
01:31:58.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:31:58 on dist-test-slave-vbz4"
01:31:58.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.137135 531 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.003s
01:31:58.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.143267 557 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:31:58.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.144510 531 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
01:31:58.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.144919 531 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-2/data,/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-2/wal
01:31:58.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef"
01:31:58.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:31:58 on dist-test-slave-vbz4"
01:31:58.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.145378 531 fs_report.cc:383] FS layout report
01:31:58.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:31:58.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-2/wal
01:31:58.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-2/wal
01:31:58.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-2/data/data
01:31:58.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:31:58.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:31:58.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:31:58.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:31:58.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:31:58.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:31:58.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:31:58.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:31:58.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:31:58.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:31:58.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:31:58.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.186902 531 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:31:58.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.188951 531 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:31:58.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.189515 531 kserver.cc:163] Server-wide thread pool size limit: 3276
01:31:58.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.277577 531 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.188:38631
01:31:58.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.277684 608 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.188:38631 every 8 connection(s)
01:31:58.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.280649 531 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-2/data/info.pb
01:31:58.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.287292 609 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:31:58.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.289422 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu as pid 531
01:31:58.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.289844 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/master-2/wal/instance
01:31:58.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.303078 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 39385 } has no permanent_uuid. Determining permanent_uuid...
01:31:58.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.336058 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } has no permanent_uuid. Determining permanent_uuid...
01:31:58.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.348691 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } has no permanent_uuid. Determining permanent_uuid...
01:31:58.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.370060 609 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef: Bootstrap starting.
01:31:58.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.378088 609 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef: Neither blocks nor log segments found. Creating new log.
01:31:58.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.380623 609 log.cc:824] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef: Log is configured to *not* fsync() on all Append() calls
01:31:58.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.387748 609 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef: No bootstrap required, opened a new log
01:31:58.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.412998 609 raft_consensus.cc:348] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 39385 } } peers { permanent_uuid: "b652e14749ba4be2ba681f496c77c6d2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } } peers { permanent_uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } }
01:31:58.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.414057 609 raft_consensus.cc:374] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:31:58.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.414318 609 raft_consensus.cc:729] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: aa82b48a569d4cdeb54cafbb59d2e2ef, State: Initialized, Role: FOLLOWER
01:31:58.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.415424 609 consensus_queue.cc:260] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 39385 } } peers { permanent_uuid: "b652e14749ba4be2ba681f496c77c6d2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } } peers { permanent_uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } }
01:31:58.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.417989 618 sys_catalog.cc:455] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 39385 } } peers { permanent_uuid: "b652e14749ba4be2ba681f496c77c6d2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } } peers { permanent_uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } } }
01:31:58.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.419099 618 sys_catalog.cc:458] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef [sys.catalog]: This master's current role is: FOLLOWER
01:31:58.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.419932 609 sys_catalog.cc:564] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef [sys.catalog]: configured and running, proceeding with master startup.
01:31:58.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:58.446540 563 tablet.cc:2369] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:31:58.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:58.450152 629 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:31:58.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:58.450836 629 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:31:58.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.443526 531 master_runner.cc:186] Error getting master registration for 127.0.93.190:39385: OK, instance_id { permanent_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e" instance_seqno: 1790645514417452 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:31:58.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.480124 531 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:31:58.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.744486 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } attempt: 7
01:31:58.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.772596 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2: Bootstrap starting.
01:31:58.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.778616 529 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2: Neither blocks nor log segments found. Creating new log.
01:31:58.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.780367 529 log.cc:824] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2: Log is configured to *not* fsync() on all Append() calls
01:31:58.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.785300 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2: No bootstrap required, opened a new log
01:31:58.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.810295 529 raft_consensus.cc:348] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 39385 } } peers { permanent_uuid: "b652e14749ba4be2ba681f496c77c6d2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } } peers { permanent_uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } }
01:31:58.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.811048 529 raft_consensus.cc:374] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:31:58.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.811249 529 raft_consensus.cc:729] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b652e14749ba4be2ba681f496c77c6d2, State: Initialized, Role: FOLLOWER
01:31:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.812193 529 consensus_queue.cc:260] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 39385 } } peers { permanent_uuid: "b652e14749ba4be2ba681f496c77c6d2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } } peers { permanent_uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } }
01:31:58.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.814815 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 39385 } } peers { permanent_uuid: "b652e14749ba4be2ba681f496c77c6d2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } } peers { permanent_uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } } }
01:31:58.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.816128 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [sys.catalog]: This master's current role is: FOLLOWER
01:31:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.817067 529 sys_catalog.cc:564] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [sys.catalog]: configured and running, proceeding with master startup.
01:31:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:58.854034 644 catalog_manager.cc:1558] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:31:58.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:58.854832 644 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:31:58.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.848054 456 master_runner.cc:186] Error getting master registration for 127.0.93.190:39385: OK, instance_id { permanent_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e" instance_seqno: 1790645514417452 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:31:58.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.885332 456 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:31:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.887795 633 raft_consensus.cc:482] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:31:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.888343 633 raft_consensus.cc:504] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 39385 } } peers { permanent_uuid: "b652e14749ba4be2ba681f496c77c6d2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } } peers { permanent_uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } }
01:31:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.890911 633 leader_election.cc:288] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 58b9e5311d8a4d10b289ae758bf1cb8e (127.0.93.190:39385), aa82b48a569d4cdeb54cafbb59d2e2ef (127.0.93.188:38631)
01:31:58.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.892400 584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b652e14749ba4be2ba681f496c77c6d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef" is_pre_election: true
01:31:58.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.893291 584 raft_consensus.cc:2454] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b652e14749ba4be2ba681f496c77c6d2 in term 0.
01:31:58.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.894475 480 leader_election.cc:302] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: aa82b48a569d4cdeb54cafbb59d2e2ef, b652e14749ba4be2ba681f496c77c6d2; no voters:
01:31:58.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.895900 633 raft_consensus.cc:2781] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [term 0 FOLLOWER]: Leader pre-election won for term 1
01:31:58.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.897900 633 raft_consensus.cc:482] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
01:31:58.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.898751 633 raft_consensus.cc:3037] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [term 0 FOLLOWER]: Advancing to term 1
01:31:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.915524 633 raft_consensus.cc:504] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 39385 } } peers { permanent_uuid: "b652e14749ba4be2ba681f496c77c6d2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } } peers { permanent_uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } }
01:31:58.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.926407 633 leader_election.cc:288] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [CANDIDATE]: Term 1 election: Requested vote from peers 58b9e5311d8a4d10b289ae758bf1cb8e (127.0.93.190:39385), aa82b48a569d4cdeb54cafbb59d2e2ef (127.0.93.188:38631)
01:31:58.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.927016 584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b652e14749ba4be2ba681f496c77c6d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef"
01:31:58.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.927733 584 raft_consensus.cc:3037] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef [term 0 FOLLOWER]: Advancing to term 1
01:31:58.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.936241 584 raft_consensus.cc:2454] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b652e14749ba4be2ba681f496c77c6d2 in term 1.
01:31:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.943324 480 leader_election.cc:302] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: aa82b48a569d4cdeb54cafbb59d2e2ef, b652e14749ba4be2ba681f496c77c6d2; no voters:
01:31:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.944525 633 raft_consensus.cc:2781] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [term 1 FOLLOWER]: Leader election won for term 1
01:31:58.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.952718 633 raft_consensus.cc:686] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [term 1 LEADER]: Becoming Leader. State: Replica: b652e14749ba4be2ba681f496c77c6d2, State: Running, Role: LEADER
01:31:58.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.953737 633 consensus_queue.cc:237] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 39385 } } peers { permanent_uuid: "b652e14749ba4be2ba681f496c77c6d2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } } peers { permanent_uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } }
01:31:58.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.974058 646 sys_catalog.cc:455] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b652e14749ba4be2ba681f496c77c6d2. Latest consensus state: current_term: 1 leader_uuid: "b652e14749ba4be2ba681f496c77c6d2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 39385 } } peers { permanent_uuid: "b652e14749ba4be2ba681f496c77c6d2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } } peers { permanent_uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } } }
01:31:58.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.975708 646 sys_catalog.cc:458] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [sys.catalog]: This master's current role is: LEADER
01:31:58.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.980513 648 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
01:31:58.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:58.988395 648 catalog_manager.cc:1484] Initializing Kudu cluster ID...
01:31:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.014951 584 raft_consensus.cc:1261] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef [term 1 FOLLOWER]: Refusing update from remote peer b652e14749ba4be2ba681f496c77c6d2: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:31:59.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.017472 633 consensus_queue.cc:1035] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
01:31:59.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:59.036830 483 tablet.cc:2369] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:31:59.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.066956 650 sys_catalog.cc:455] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef [sys.catalog]: SysCatalogTable state changed. Reason: New leader b652e14749ba4be2ba681f496c77c6d2. Latest consensus state: current_term: 1 leader_uuid: "b652e14749ba4be2ba681f496c77c6d2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 39385 } } peers { permanent_uuid: "b652e14749ba4be2ba681f496c77c6d2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } } peers { permanent_uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } } }
01:31:59.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.067880 650 sys_catalog.cc:458] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef [sys.catalog]: This master's current role is: FOLLOWER
01:31:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.081950 650 sys_catalog.cc:455] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "b652e14749ba4be2ba681f496c77c6d2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 39385 } } peers { permanent_uuid: "b652e14749ba4be2ba681f496c77c6d2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } } peers { permanent_uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } } }
01:31:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.082710 650 sys_catalog.cc:458] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef [sys.catalog]: This master's current role is: FOLLOWER
01:31:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.083176 651 mvcc.cc:205] Tried to move back new op lower bound from 7334484045862637568 to 7334484045648863232. Current Snapshot: MvccSnapshot[applied={T|T < 7334484045862637568}]
01:31:59.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.086966 646 sys_catalog.cc:455] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "b652e14749ba4be2ba681f496c77c6d2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 39385 } } peers { permanent_uuid: "b652e14749ba4be2ba681f496c77c6d2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } } peers { permanent_uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } } }
01:31:59.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.087764 646 sys_catalog.cc:458] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [sys.catalog]: This master's current role is: LEADER
01:31:59.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.094296 648 catalog_manager.cc:1347] Generated new cluster ID: 4b3459fabbfd42c083d7768f3e37cc11
01:31:59.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.094630 648 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
01:31:59.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.099865 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } attempt: 7
01:31:59.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.149938 648 catalog_manager.cc:1370] Generated new certificate authority record
01:31:59.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.153424 648 catalog_manager.cc:1504] Loading token signing keys...
01:31:59.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.156435 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 58b9e5311d8a4d10b289ae758bf1cb8e: Bootstrap starting.
01:31:59.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.184795 648 catalog_manager.cc:5868] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2: Generated new TSK 0
01:31:59.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.186851 648 catalog_manager.cc:1514] Initializing in-progress tserver states...
01:31:59.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.186937 454 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 58b9e5311d8a4d10b289ae758bf1cb8e: Neither blocks nor log segments found. Creating new log.
01:31:59.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.194155 454 log.cc:824] T 00000000000000000000000000000000 P 58b9e5311d8a4d10b289ae758bf1cb8e: Log is configured to *not* fsync() on all Append() calls
01:31:59.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.210338 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 58b9e5311d8a4d10b289ae758bf1cb8e: No bootstrap required, opened a new log
01:31:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.213513 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:31:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:31:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-0/wal
01:31:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-0/data
01:31:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:31:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:31:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:31:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:31:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:31:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:31:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-0/logs
01:31:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-0/data/info.pb
01:31:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:31:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:31:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:31:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:31:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:31:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:31:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:31:59.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:31:59.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:31:59.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:31:59.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
01:31:59.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
01:31:59.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
01:31:59.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:31:59.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:39385,127.0.93.189:36365,127.0.93.188:38631
01:31:59.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:38785
01:31:59.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:31:59.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:31:59.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:31:59.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.245785 454 raft_consensus.cc:348] T 00000000000000000000000000000000 P 58b9e5311d8a4d10b289ae758bf1cb8e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 39385 } } peers { permanent_uuid: "b652e14749ba4be2ba681f496c77c6d2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } } peers { permanent_uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } }
01:31:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.247046 454 raft_consensus.cc:374] T 00000000000000000000000000000000 P 58b9e5311d8a4d10b289ae758bf1cb8e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:31:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.247576 454 raft_consensus.cc:729] T 00000000000000000000000000000000 P 58b9e5311d8a4d10b289ae758bf1cb8e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 58b9e5311d8a4d10b289ae758bf1cb8e, State: Initialized, Role: FOLLOWER
01:31:59.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.248756 454 consensus_queue.cc:260] T 00000000000000000000000000000000 P 58b9e5311d8a4d10b289ae758bf1cb8e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 39385 } } peers { permanent_uuid: "b652e14749ba4be2ba681f496c77c6d2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } } peers { permanent_uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } }
01:31:59.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.260597 658 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58b9e5311d8a4d10b289ae758bf1cb8e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 39385 } } peers { permanent_uuid: "b652e14749ba4be2ba681f496c77c6d2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } } peers { permanent_uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } } }
01:31:59.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.261508 658 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58b9e5311d8a4d10b289ae758bf1cb8e [sys.catalog]: This master's current role is: FOLLOWER
01:31:59.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.264514 454 sys_catalog.cc:564] T 00000000000000000000000000000000 P 58b9e5311d8a4d10b289ae758bf1cb8e [sys.catalog]: configured and running, proceeding with master startup.
01:31:59.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.264679 429 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b652e14749ba4be2ba681f496c77c6d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e" is_pre_election: true
01:31:59.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.265626 429 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 58b9e5311d8a4d10b289ae758bf1cb8e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b652e14749ba4be2ba681f496c77c6d2 in term 0.
01:31:59.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.268431 427 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 58b9e5311d8a4d10b289ae758bf1cb8e [term 0 FOLLOWER]: Advancing to term 1
01:31:59.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.271552 428 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b652e14749ba4be2ba681f496c77c6d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e"
01:31:59.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.279485 427 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 58b9e5311d8a4d10b289ae758bf1cb8e [term 1 FOLLOWER]: Refusing update from remote peer b652e14749ba4be2ba681f496c77c6d2: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
01:31:59.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.282814 646 consensus_queue.cc:1035] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 39385 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
01:31:59.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.377053 666 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58b9e5311d8a4d10b289ae758bf1cb8e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "b652e14749ba4be2ba681f496c77c6d2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 39385 } } peers { permanent_uuid: "b652e14749ba4be2ba681f496c77c6d2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } } peers { permanent_uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } } }
01:31:59.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.376966 658 sys_catalog.cc:455] T 00000000000000000000000000000000 P 58b9e5311d8a4d10b289ae758bf1cb8e [sys.catalog]: SysCatalogTable state changed. Reason: New leader b652e14749ba4be2ba681f496c77c6d2. Latest consensus state: current_term: 1 leader_uuid: "b652e14749ba4be2ba681f496c77c6d2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 39385 } } peers { permanent_uuid: "b652e14749ba4be2ba681f496c77c6d2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } } peers { permanent_uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } } }
01:31:59.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.378460 666 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58b9e5311d8a4d10b289ae758bf1cb8e [sys.catalog]: This master's current role is: FOLLOWER
01:31:59.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.378381 658 sys_catalog.cc:458] T 00000000000000000000000000000000 P 58b9e5311d8a4d10b289ae758bf1cb8e [sys.catalog]: This master's current role is: FOLLOWER
01:31:59.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.409696 674 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
01:31:59.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.458063 629 catalog_manager.cc:1259] Loaded cluster ID: 4b3459fabbfd42c083d7768f3e37cc11
01:31:59.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.458791 629 catalog_manager.cc:1552] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef: loading cluster ID for follower catalog manager: success
01:31:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.476954 629 catalog_manager.cc:1574] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef: acquiring CA information for follower catalog manager: success
01:31:59.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.493403 629 catalog_manager.cc:1602] T 00000000000000000000000000000000 P aa82b48a569d4cdeb54cafbb59d2e2ef: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:31:59.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.793011 646 sys_catalog.cc:455] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "b652e14749ba4be2ba681f496c77c6d2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "58b9e5311d8a4d10b289ae758bf1cb8e" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 39385 } } peers { permanent_uuid: "b652e14749ba4be2ba681f496c77c6d2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 36365 } } peers { permanent_uuid: "aa82b48a569d4cdeb54cafbb59d2e2ef" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38631 } } }
01:31:59.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.794739 646 sys_catalog.cc:458] T 00000000000000000000000000000000 P b652e14749ba4be2ba681f496c77c6d2 [sys.catalog]: This master's current role is: LEADER
01:31:59.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:59.864540 657 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:31:59.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:59.865753 657 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:31:59.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:59.866534 657 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:31:59.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:59.928061 657 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:31:59.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:31:59.930037 657 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.129
01:31:59.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.998051 657 tablet_server_runner.cc:78] Tablet server non-default flags:
01:31:59.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:31:59.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:38785
01:31:59.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:31:59.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:31:59.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-0/data
01:31:59.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-0/wal
01:31:59.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:31:59.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:31:59.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
01:31:59.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:31:59.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:31:59.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:31:59.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-0/data/info.pb
01:31:59.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
01:31:59.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:31:59.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:39385,127.0.93.189:36365,127.0.93.188:38631
01:31:59.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:31:59.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:31:59.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:31:59.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:32:00.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:32:00.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
01:32:00.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-0/logs
01:32:00.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:32:00.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:32:00.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:32:00.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:32:00.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:32:00.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:32:00.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:32:00.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:32:00.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:32:00.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:32:00.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:31:59.999730 657 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:32:00.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:00.001979 657 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:32:00.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:00.029249 682 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:00.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:00.426759 674 catalog_manager.cc:1259] Loaded cluster ID: 4b3459fabbfd42c083d7768f3e37cc11
01:32:00.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:00.427055 674 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 58b9e5311d8a4d10b289ae758bf1cb8e: loading cluster ID for follower catalog manager: success
01:32:00.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:00.451516 674 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 58b9e5311d8a4d10b289ae758bf1cb8e: acquiring CA information for follower catalog manager: success
01:32:00.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:00.468698 674 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 58b9e5311d8a4d10b289ae758bf1cb8e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
01:32:02.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:01.421078 681 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 657
01:32:02.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:00.031109 683 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:02.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:02.755069 681 kernel_stack_watchdog.cc:198] Thread 657 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 390ms:
01:32:02.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
01:32:02.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
01:32:02.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
01:32:02.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
01:32:02.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:02.756368 657 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.728s user 0.671s sys 1.162s
01:32:02.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:02.759696 657 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.731s user 0.672s sys 1.162s
01:32:02.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:02.763237 685 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:02.768577 684 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2725 milliseconds
01:32:02.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.769543 657 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:32:02.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.771564 657 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:32:02.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.775837 657 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:32:02.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.780228 657 hybrid_clock.cc:648] HybridClock initialized: now 1790645522780160 us; error 55 us; skew 500 ppm
01:32:02.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.783694 657 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:32:02.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.793609 657 webserver.cc:460] Webserver started at http://127.0.93.129:42131/ using document root <none> and password file <none>
01:32:02.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.797327 657 fs_manager.cc:359] Metadata directory not provided
01:32:02.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.798003 657 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:32:02.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.798947 657 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:32:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.808732 657 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-0/data/instance:
01:32:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "21b7e2826e70451fb7e1931408319811"
01:32:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:32:02 on dist-test-slave-vbz4"
01:32:02.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.811098 657 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-0/wal/instance:
01:32:02.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "21b7e2826e70451fb7e1931408319811"
01:32:02.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:32:02 on dist-test-slave-vbz4"
01:32:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.822934 657 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.004s sys 0.006s
01:32:02.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.830734 696 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:32:02.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.832715 657 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.003s sys 0.004s
01:32:02.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.833261 657 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-0/data,/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-0/wal
01:32:02.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "21b7e2826e70451fb7e1931408319811"
01:32:02.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:32:02 on dist-test-slave-vbz4"
01:32:02.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.834233 657 fs_report.cc:383] FS layout report
01:32:02.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:32:02.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-0/wal
01:32:02.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-0/wal
01:32:02.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-0/data/data
01:32:02.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:32:02.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:32:02.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:32:02.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:32:02.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:32:02.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:32:02.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:32:02.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:32:02.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:32:02.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:32:02.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:32:02.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.882148 657 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:32:02.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.884138 657 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:32:02.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.884704 657 kserver.cc:163] Server-wide thread pool size limit: 3276
01:32:02.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.889460 657 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:32:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.896955 657 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:32:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.897254 657 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:32:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.897537 657 ts_tablet_manager.cc:610] Registered 0 tablets
01:32:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:02.897739 657 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:32:03.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:03.110018 657 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.129:42129
01:32:03.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:03.110201 809 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.129:42129 every 8 connection(s)
01:32:03.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:03.113126 657 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-0/data/info.pb
01:32:03.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:03.123083 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu as pid 657
01:32:03.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:03.124071 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-0/wal/instance
01:32:03.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:03.143366 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:32:03.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:32:03.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-1/wal
01:32:03.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-1/data
01:32:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:32:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:32:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:32:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:32:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:32:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:32:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-1/logs
01:32:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-1/data/info.pb
01:32:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:32:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:32:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:32:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:32:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:32:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:32:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:32:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:32:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:32:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:32:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
01:32:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
01:32:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
01:32:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:32:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:39385,127.0.93.189:36365,127.0.93.188:38631
01:32:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:38785
01:32:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:32:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:32:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:32:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:03.208666 811 heartbeater.cc:347] Connected to a master server at 127.0.93.189:36365
01:32:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:03.209177 811 heartbeater.cc:464] Registering TS with master...
01:32:03.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:03.210448 811 heartbeater.cc:511] Master 127.0.93.189:36365 requested a full tablet report, sending...
01:32:03.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:03.213522 494 ts_manager.cc:194] Registered new tserver with Master: 21b7e2826e70451fb7e1931408319811 (127.0.93.129:42129)
01:32:03.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:03.217724 494 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.129:57799
01:32:03.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:03.223399 810 heartbeater.cc:347] Connected to a master server at 127.0.93.188:38631
01:32:03.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:03.223820 810 heartbeater.cc:464] Registering TS with master...
01:32:03.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:03.224493 810 heartbeater.cc:511] Master 127.0.93.188:38631 requested a full tablet report, sending...
01:32:03.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:03.225525 814 heartbeater.cc:347] Connected to a master server at 127.0.93.190:39385
01:32:03.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:03.225855 814 heartbeater.cc:464] Registering TS with master...
01:32:03.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:03.226476 814 heartbeater.cc:511] Master 127.0.93.190:39385 requested a full tablet report, sending...
01:32:03.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:03.227370 574 ts_manager.cc:194] Registered new tserver with Master: 21b7e2826e70451fb7e1931408319811 (127.0.93.129:42129)
01:32:03.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:03.229852 419 ts_manager.cc:194] Registered new tserver with Master: 21b7e2826e70451fb7e1931408319811 (127.0.93.129:42129)
01:32:03.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:03.577111 818 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:32:03.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:03.577598 818 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:32:03.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:03.578022 818 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:32:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:03.610442 818 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:32:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:03.611502 818 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.130
01:32:03.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:03.649592 818 tablet_server_runner.cc:78] Tablet server non-default flags:
01:32:03.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:32:03.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:38785
01:32:03.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:32:03.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:32:03.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-1/data
01:32:03.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-1/wal
01:32:03.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:32:03.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:32:03.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
01:32:03.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:32:03.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:32:03.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:32:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-1/data/info.pb
01:32:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
01:32:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:32:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:39385,127.0.93.189:36365,127.0.93.188:38631
01:32:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:32:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:32:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:32:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:32:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:32:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
01:32:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-1/logs
01:32:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:32:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:32:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:32:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:32:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:32:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:32:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:32:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:32:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:32:03.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:32:03.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:03.650765 818 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:32:03.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:03.652514 818 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:32:03.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:03.679153 830 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:04.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:04.223413 811 heartbeater.cc:503] Master 127.0.93.189:36365 was elected leader, sending a full tablet report...
01:32:05.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:03.679320 827 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:05.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:03.680231 828 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:05.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:05.024159 829 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1341 milliseconds
01:32:05.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.024266 818 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:32:05.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.025544 818 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:32:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.027810 818 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:32:05.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.029225 818 hybrid_clock.cc:648] HybridClock initialized: now 1790645525029193 us; error 68 us; skew 500 ppm
01:32:05.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.030158 818 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:32:05.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.037088 818 webserver.cc:460] Webserver started at http://127.0.93.130:42591/ using document root <none> and password file <none>
01:32:05.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.038110 818 fs_manager.cc:359] Metadata directory not provided
01:32:05.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.038318 818 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:32:05.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.038697 818 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:32:05.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.043690 818 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-1/data/instance:
01:32:05.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c64a74a2e4c14e6386ac4a0443f3d285"
01:32:05.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:32:05 on dist-test-slave-vbz4"
01:32:05.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.044808 818 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-1/wal/instance:
01:32:05.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c64a74a2e4c14e6386ac4a0443f3d285"
01:32:05.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:32:05 on dist-test-slave-vbz4"
01:32:05.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.052160 818 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
01:32:05.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.057972 838 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:32:05.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.059036 818 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
01:32:05.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.059427 818 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-1/data,/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-1/wal
01:32:05.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c64a74a2e4c14e6386ac4a0443f3d285"
01:32:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:32:05 on dist-test-slave-vbz4"
01:32:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.059842 818 fs_report.cc:383] FS layout report
01:32:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:32:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-1/wal
01:32:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-1/wal
01:32:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-1/data/data
01:32:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:32:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:32:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:32:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:32:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:32:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:32:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:32:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:32:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:32:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:32:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:32:05.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.109414 818 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:32:05.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.111145 818 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:32:05.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.111781 818 kserver.cc:163] Server-wide thread pool size limit: 3276
01:32:05.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.115566 818 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:32:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.120414 818 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:32:05.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.120893 818 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.001s user 0.000s sys 0.000s
01:32:05.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.121296 818 ts_tablet_manager.cc:610] Registered 0 tablets
01:32:05.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.121490 818 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:32:05.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.282502 818 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.130:42245
01:32:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.282569 950 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.130:42245 every 8 connection(s)
01:32:05.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.285995 818 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-1/data/info.pb
01:32:05.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.293766 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu as pid 818
01:32:05.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.294333 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-1/wal/instance
01:32:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.332382 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:32:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:32:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-2/wal
01:32:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-2/data
01:32:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:32:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:32:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:32:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:32:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:32:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:32:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-2/logs
01:32:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-2/data/info.pb
01:32:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:32:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:32:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:32:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:32:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:32:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:32:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:32:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:32:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
01:32:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:32:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
01:32:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
01:32:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
01:32:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:32:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:39385,127.0.93.189:36365,127.0.93.188:38631
01:32:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:38785
01:32:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:32:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:32:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
01:32:05.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.355742 952 heartbeater.cc:347] Connected to a master server at 127.0.93.189:36365
01:32:05.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.356251 952 heartbeater.cc:464] Registering TS with master...
01:32:05.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.357465 952 heartbeater.cc:511] Master 127.0.93.189:36365 requested a full tablet report, sending...
01:32:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.360679 494 ts_manager.cc:194] Registered new tserver with Master: c64a74a2e4c14e6386ac4a0443f3d285 (127.0.93.130:42245)
01:32:05.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.363307 494 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.130:53037
01:32:05.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.370095 953 heartbeater.cc:347] Connected to a master server at 127.0.93.190:39385
01:32:05.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.370512 953 heartbeater.cc:464] Registering TS with master...
01:32:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.371335 953 heartbeater.cc:511] Master 127.0.93.190:39385 requested a full tablet report, sending...
01:32:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.371522 951 heartbeater.cc:347] Connected to a master server at 127.0.93.188:38631
01:32:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.371840 951 heartbeater.cc:464] Registering TS with master...
01:32:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.372905 951 heartbeater.cc:511] Master 127.0.93.188:38631 requested a full tablet report, sending...
01:32:05.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.374615 419 ts_manager.cc:194] Registered new tserver with Master: c64a74a2e4c14e6386ac4a0443f3d285 (127.0.93.130:42245)
01:32:05.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.375599 574 ts_manager.cc:194] Registered new tserver with Master: c64a74a2e4c14e6386ac4a0443f3d285 (127.0.93.130:42245)
01:32:05.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:05.713878 961 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:32:05.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:05.714447 961 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:32:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:05.715351 961 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:32:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:05.750528 961 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:32:05.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:05.751710 961 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.131
01:32:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.793635 961 tablet_server_runner.cc:78] Tablet server non-default flags:
01:32:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:32:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:38785
01:32:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:32:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:32:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-2/data
01:32:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-2/wal
01:32:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:32:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:32:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
01:32:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:32:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:32:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:32:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-2/data/info.pb
01:32:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
01:32:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:32:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:39385,127.0.93.189:36365,127.0.93.188:38631
01:32:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:32:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:32:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:32:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:32:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:32:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
01:32:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-2/logs
01:32:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:32:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:32:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:32:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
01:32:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:32:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:32:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:32:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:32:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:32:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:32:05.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.795871 961 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:32:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.798463 961 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:32:05.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:05.823371 968 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:06.369763 952 heartbeater.cc:503] Master 127.0.93.189:36365 was elected leader, sending a full tablet report...
01:32:07.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:05.823468 967 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:07.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:05.823406 970 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:07.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:05.824525 961 server_base.cc:1034] running on GCE node
01:32:07.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.168644 961 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:32:07.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.172384 961 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:32:07.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.173929 961 hybrid_clock.cc:648] HybridClock initialized: now 1790645527173859 us; error 96 us; skew 500 ppm
01:32:07.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.174968 961 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:32:07.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.184078 961 webserver.cc:460] Webserver started at http://127.0.93.131:43987/ using document root <none> and password file <none>
01:32:07.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.187355 961 fs_manager.cc:359] Metadata directory not provided
01:32:07.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.187886 961 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:32:07.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.189003 961 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:32:07.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.195521 961 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-2/data/instance:
01:32:07.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7cacedd7f65e467d8c28c763883b49ff"
01:32:07.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:32:07 on dist-test-slave-vbz4"
01:32:07.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.196805 961 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-2/wal/instance:
01:32:07.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7cacedd7f65e467d8c28c763883b49ff"
01:32:07.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:32:07 on dist-test-slave-vbz4"
01:32:07.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.205895 961 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
01:32:07.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.214020 978 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:32:07.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.215334 961 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
01:32:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.215760 961 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-2/data,/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-2/wal
01:32:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7cacedd7f65e467d8c28c763883b49ff"
01:32:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:32:07 on dist-test-slave-vbz4"
01:32:07.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.216214 961 fs_report.cc:383] FS layout report
01:32:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:32:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-2/wal
01:32:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-2/wal
01:32:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-2/data/data
01:32:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:32:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:32:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:32:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:32:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:32:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:32:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:32:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:32:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:32:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:32:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:32:07.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.268548 961 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:32:07.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.270395 961 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:32:07.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.270968 961 kserver.cc:163] Server-wide thread pool size limit: 3276
01:32:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.274636 961 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
01:32:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.281679 961 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
01:32:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.281983 961 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
01:32:07.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.282531 961 ts_tablet_manager.cc:610] Registered 0 tablets
01:32:07.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.282797 961 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
01:32:07.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.500497 961 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.131:38149
01:32:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.500888 1090 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.131:38149 every 8 connection(s)
01:32:07.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.504005 961 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-2/data/info.pb
01:32:07.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.508742 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu as pid 961
01:32:07.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.509296 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster5902766993428311575/ts-2/wal/instance
01:32:07.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.563249 1091 heartbeater.cc:347] Connected to a master server at 127.0.93.188:38631
01:32:07.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.564332 1091 heartbeater.cc:464] Registering TS with master...
01:32:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.565728 1091 heartbeater.cc:511] Master 127.0.93.188:38631 requested a full tablet report, sending...
01:32:07.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.569794 574 ts_manager.cc:194] Registered new tserver with Master: 7cacedd7f65e467d8c28c763883b49ff (127.0.93.131:38149)
01:32:07.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.571766 1093 heartbeater.cc:347] Connected to a master server at 127.0.93.190:39385
01:32:07.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.572131 1093 heartbeater.cc:464] Registering TS with master...
01:32:07.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.572758 1092 heartbeater.cc:347] Connected to a master server at 127.0.93.189:36365
01:32:07.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.573089 1092 heartbeater.cc:464] Registering TS with master...
01:32:07.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.573005 1093 heartbeater.cc:511] Master 127.0.93.190:39385 requested a full tablet report, sending...
01:32:07.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.573931 1092 heartbeater.cc:511] Master 127.0.93.189:36365 requested a full tablet report, sending...
01:32:07.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.575843 419 ts_manager.cc:194] Registered new tserver with Master: 7cacedd7f65e467d8c28c763883b49ff (127.0.93.131:38149)
01:32:07.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.575897 494 ts_manager.cc:194] Registered new tserver with Master: 7cacedd7f65e467d8c28c763883b49ff (127.0.93.131:38149)
01:32:07.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.578236 494 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.131:36427
01:32:07.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:07.601289 374 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
01:32:07.615 [DEBUG - main] (MiniKuduCluster.java:195) Response:
01:32:07.616 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
01:32:07.632 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.0.93.190"
port: 39385
}
bound_http_address {
host: "127.0.93.190"
port: 39651
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.0.93.189"
port: 36365
}
bound_http_address {
host: "127.0.93.189"
port: 45489
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.0.93.188"
port: 38631
}
bound_http_address {
host: "127.0.93.188"
port: 37149
}
}
}
01:32:07.638 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
01:32:07.642 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.0.93.129"
port: 42129
}
bound_http_address {
host: "127.0.93.129"
port: 42131
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.0.93.130"
port: 42245
}
bound_http_address {
host: "127.0.93.130"
port: 42591
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.0.93.131"
port: 38149
}
bound_http_address {
host: "127.0.93.131"
port: 43987
}
}
}
01:32:07.642 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
01:32:07.866 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
01:32:07.994 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskCdr2xp/test-tmp
01:32:07.994 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
01:32:07.995 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskCdr2xp/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
01:32:07.998 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
01:32:08.387 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:32:08.387 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
01:32:08.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:08.391728 1102 mini_oidc.cc:152] Starting JWKS server
01:32:08.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:08.395537 1102 webserver.cc:460] Webserver started at http://127.0.0.1:38635/ using document root <none> and password file <none>
01:32:08.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:08.395980 1102 mini_oidc.cc:189] Starting OIDC Discovery server
01:32:08.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:08.397743 1102 webserver.cc:460] Webserver started at http://127.0.0.1:39825/ using document root <none> and password file <none>
01:32:08.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
01:32:08.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/principal' for realm 'KRBTEST.COM',
01:32:08.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
01:32:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:08 dist-test-slave-vbz4 krb5kdc[1114](info): setting up network...
01:32:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
01:32:08.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:08 dist-test-slave-vbz4 krb5kdc[1114](info): set up 2 sockets
01:32:08.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:08 dist-test-slave-vbz4 krb5kdc[1114](info): commencing operation
01:32:08.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
01:32:08.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:08.582667 1092 heartbeater.cc:503] Master 127.0.93.189:36365 was elected leader, sending a full tablet report...
01:32:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:09.607908 1102 mini_kdc.cc:121] Time spent starting KDC: real 1.185s user 0.001s sys 0.023s
01:32:09.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
01:32:09.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:32:09.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
01:32:09.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
01:32:09.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:32:09.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
01:32:09.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
01:32:09.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:32:09.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
01:32:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
01:32:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/test-user.keytab.
01:32:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/test-user.keytab.
01:32:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:09 dist-test-slave-vbz4 krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790645529, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:32:09.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:32:09.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T01:32:09Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
01:32:09.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T01:32:09Z Disabled control of system clock
01:32:09.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.190@KRBTEST.COM; defaulting to no policy
01:32:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:32:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.190@KRBTEST.COM" created.
01:32:09.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:32:09.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kudu_127.1.19.190.keytab.
01:32:09.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kudu_127.1.19.190.keytab.
01:32:09.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:09.836989 1102 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:32:09.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:32:09.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/wal
01:32:09.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/data
01:32:09.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:32:09.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:32:09.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:32:09.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:32:09.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:32:09.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:32:09.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/logs
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/data/info.pb
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.190:45973
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.190
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kudu_127.1.19.190.keytab
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.190
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:43983
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:32:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:32:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:32:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:32:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:45973,127.1.19.189:43805,127.1.19.188:46459
01:32:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:32:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38635/jwks.json
01:32:09.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:32:10.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:10.214897 1128 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:32:10.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:10.215631 1128 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:32:10.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:10.216272 1128 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:32:10.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:10.255831 1128 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:32:10.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:10.256071 1128 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:32:10.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:10.256747 1128 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:32:10.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:10.256981 1128 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:32:10.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:10.257180 1128 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:32:10.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:10.257372 1128 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:32:10.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:10.257618 1128 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38635/jwks.json
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:10.305665 1128 master_runner.cc:386] Master server non-default flags:
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:43983
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/data
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/wal
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:45973,127.1.19.189:43805,127.1.19.188:46459
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kudu_127.1.19.190.keytab
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.190
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.190:45973
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38635/jwks.json
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/data/info.pb
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.190
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:32:10.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:32:10.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:32:10.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/logs
01:32:10.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:32:10.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:32:10.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:32:10.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:32:10.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:32:10.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:32:10.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:32:10.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:32:10.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:32:10.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:32:10.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:10.306958 1128 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:32:10.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:10.308128 1128 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:32:10.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:10.330977 1134 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:11.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:10.331566 1135 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:11.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:10.333722 1128 server_base.cc:1034] running on GCE node
01:32:11.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:10.332273 1137 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:11.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.670624 1128 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:32:11.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.673553 1128 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:32:11.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.675021 1128 hybrid_clock.cc:648] HybridClock initialized: now 1790645531674976 us; error 87 us; skew 500 ppm
01:32:11.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.676178 1128 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:32:11.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:11 dist-test-slave-vbz4 krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790645531, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:32:11.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.683334 1128 init.cc:377] Logged in from keytab as kudu/127.1.19.190@KRBTEST.COM (short username kudu)
01:32:11.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.689606 1128 webserver.cc:460] Webserver started at http://127.1.19.190:39973/ using document root <none> and password file <none>
01:32:11.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.690549 1128 fs_manager.cc:359] Metadata directory not provided
01:32:11.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.690754 1128 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:32:11.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.691136 1128 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:32:11.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.696422 1128 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/data/instance:
01:32:11.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea15bad27d404c6da448f64733d59bea"
01:32:11.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:32:11 on dist-test-slave-vbz4"
01:32:11.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.697569 1128 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/wal/instance:
01:32:11.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea15bad27d404c6da448f64733d59bea"
01:32:11.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:32:11 on dist-test-slave-vbz4"
01:32:11.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.705435 1128 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.003s
01:32:11.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.711496 1145 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:32:11.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.712888 1128 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
01:32:11.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.713195 1128 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/data,/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/wal
01:32:11.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea15bad27d404c6da448f64733d59bea"
01:32:11.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:32:11 on dist-test-slave-vbz4"
01:32:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.713533 1128 fs_report.cc:383] FS layout report
01:32:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:32:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/wal
01:32:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/wal
01:32:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/data/data
01:32:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:32:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:32:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:32:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:32:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:32:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:32:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:32:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:32:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:32:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:32:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:32:11.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.757942 1128 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:32:11.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.765220 1128 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:32:11.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.766281 1128 kserver.cc:163] Server-wide thread pool size limit: 3276
01:32:11.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.860883 1128 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.190:45973
01:32:11.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.860942 1196 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.190:45973 every 8 connection(s)
01:32:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.864074 1128 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/data/info.pb
01:32:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.870500 1102 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu as pid 1128
01:32:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.870873 1197 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:32:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.870996 1102 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/wal/instance
01:32:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.884315 1197 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.190" port: 45973 } has no permanent_uuid. Determining permanent_uuid...
01:32:11.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.189@KRBTEST.COM; defaulting to no policy
01:32:11.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:32:11.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.189@KRBTEST.COM" created.
01:32:11.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:11 dist-test-slave-vbz4 krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790645531, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.190@KRBTEST.COM for kudu/127.1.19.190@KRBTEST.COM
01:32:11.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:32:11.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kudu_127.1.19.189.keytab.
01:32:11.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kudu_127.1.19.189.keytab.
01:32:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.937733 1102 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:32:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:32:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/wal
01:32:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/data
01:32:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:32:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:32:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:32:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:32:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:32:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:32:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/logs
01:32:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/data/info.pb
01:32:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:32:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.189:43805
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.189
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kudu_127.1.19.189.keytab
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.189
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:43983
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:45973,127.1.19.189:43805,127.1.19.188:46459
01:32:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:32:11.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38635/jwks.json
01:32:11.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:32:11.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.941545 1197 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.189" port: 43805 } has no permanent_uuid. Determining permanent_uuid...
01:32:11.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:11.945434 1149 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.189:43805: connect: Connection refused (error 111)
01:32:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:11.950186 1197 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.189:43805: Network error: Client connection negotiation failed: client connection to 127.1.19.189:43805: connect: Connection refused (error 111)
01:32:12.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:11.999776 1197 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.189" port: 43805 } attempt: 1
01:32:12.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:12.004700 1197 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.189:43805: Network error: Client connection negotiation failed: client connection to 127.1.19.189:43805: connect: Connection refused (error 111)
01:32:12.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:12.073519 1197 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.189" port: 43805 } attempt: 2
01:32:12.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:12.077183 1197 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.189:43805: Network error: Client connection negotiation failed: client connection to 127.1.19.189:43805: connect: Connection refused (error 111)
01:32:12.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:12.168674 1197 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.189" port: 43805 } attempt: 3
01:32:12.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:12.173352 1197 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.189:43805: Network error: Client connection negotiation failed: client connection to 127.1.19.189:43805: connect: Connection refused (error 111)
01:32:12.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:12.306766 1203 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:32:12.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:12.307432 1203 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:32:12.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:12.308103 1203 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:32:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:12.316807 1197 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.189" port: 43805 } attempt: 4
01:32:12.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:12.322053 1197 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.189:43805: Network error: Client connection negotiation failed: client connection to 127.1.19.189:43805: connect: Connection refused (error 111)
01:32:12.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:12.344115 1203 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:32:12.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:12.344492 1203 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:32:12.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:12.344868 1203 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:32:12.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:12.345096 1203 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:32:12.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:12.345309 1203 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:32:12.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:12.345510 1203 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:32:12.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:12.345656 1203 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38635/jwks.json
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:12.399461 1203 master_runner.cc:386] Master server non-default flags:
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:43983
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/data
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/wal
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:45973,127.1.19.189:43805,127.1.19.188:46459
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kudu_127.1.19.189.keytab
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.189
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.189:43805
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38635/jwks.json
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/data/info.pb
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.189
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/logs
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:32:12.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:32:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:32:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:32:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:32:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:32:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:32:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:32:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:32:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:32:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:12.400852 1203 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:32:12.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:12.401968 1203 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:32:12.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:12.423933 1211 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:12.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:12.621480 1197 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.189" port: 43805 } attempt: 5
01:32:12.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:12.627029 1197 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.189:43805: Network error: Client connection negotiation failed: client connection to 127.1.19.189:43805: connect: Connection refused (error 111)
01:32:13.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:13.174503 1197 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.189" port: 43805 } attempt: 6
01:32:13.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:13.182117 1197 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.189:43805: Network error: Client connection negotiation failed: client connection to 127.1.19.189:43805: connect: Connection refused (error 111)
01:32:13.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:12.424613 1210 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:13.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:12.424626 1213 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:13.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:12.426424 1203 server_base.cc:1034] running on GCE node
01:32:13.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:13.855225 1203 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:32:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:13.858385 1203 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:32:13.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:13.860278 1203 hybrid_clock.cc:648] HybridClock initialized: now 1790645533860197 us; error 100 us; skew 500 ppm
01:32:13.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:13.861476 1203 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:32:13.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:13 dist-test-slave-vbz4 krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790645533, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:32:13.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:13.878662 1203 init.cc:377] Logged in from keytab as kudu/127.1.19.189@KRBTEST.COM (short username kudu)
01:32:13.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:13.884479 1203 webserver.cc:460] Webserver started at http://127.1.19.189:45633/ using document root <none> and password file <none>
01:32:13.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:13.885578 1203 fs_manager.cc:359] Metadata directory not provided
01:32:13.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:13.885850 1203 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:32:13.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:13.886636 1203 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:32:13.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:13.894107 1203 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/data/instance:
01:32:13.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3184c3d2e2de4b37ae9a8742f74bd860"
01:32:13.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:32:13 on dist-test-slave-vbz4"
01:32:13.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:13.895654 1203 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/wal/instance:
01:32:13.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3184c3d2e2de4b37ae9a8742f74bd860"
01:32:13.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:32:13 on dist-test-slave-vbz4"
01:32:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:13.905346 1203 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.007s sys 0.001s
01:32:13.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:13.914942 1226 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:32:13.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:13.916481 1203 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.005s sys 0.001s
01:32:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:13.916833 1203 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/data,/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/wal
01:32:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3184c3d2e2de4b37ae9a8742f74bd860"
01:32:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:32:13 on dist-test-slave-vbz4"
01:32:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:13.917153 1203 fs_report.cc:383] FS layout report
01:32:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:32:13.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/wal
01:32:13.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/wal
01:32:13.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/data/data
01:32:13.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:32:13.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:32:13.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:32:13.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:32:13.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:32:13.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:32:13.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:32:13.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:32:13.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:32:13.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:32:13.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:32:13.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:13.955062 1203 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:32:13.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:13.962999 1203 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:32:13.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:13.963865 1203 kserver.cc:163] Server-wide thread pool size limit: 3276
01:32:14.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.073731 1203 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.189:43805
01:32:14.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.073995 1277 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.189:43805 every 8 connection(s)
01:32:14.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.076759 1203 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/data/info.pb
01:32:14.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.082904 1102 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu as pid 1203
01:32:14.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.083630 1102 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/wal/instance
01:32:14.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.084673 1278 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:32:14.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.099082 1278 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.190" port: 45973 } has no permanent_uuid. Determining permanent_uuid...
01:32:14.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.188@KRBTEST.COM; defaulting to no policy
01:32:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:32:14.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.188@KRBTEST.COM" created.
01:32:14.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:14 dist-test-slave-vbz4 krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790645533, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.189@KRBTEST.COM for kudu/127.1.19.190@KRBTEST.COM
01:32:14.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
01:32:14.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kudu_127.1.19.188.keytab.
01:32:14.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kudu_127.1.19.188.keytab.
01:32:14.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.149834 1102 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:32:14.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:32:14.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/wal
01:32:14.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/data
01:32:14.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:32:14.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:32:14.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:32:14.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:32:14.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:32:14.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:32:14.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/logs
01:32:14.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/data/info.pb
01:32:14.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:32:14.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:32:14.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:32:14.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:32:14.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:32:14.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:32:14.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:32:14.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:32:14.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:32:14.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:32:14.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:32:14.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:32:14.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.188:46459
01:32:14.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.188
01:32:14.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:32:14.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kudu_127.1.19.188.keytab
01:32:14.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.188
01:32:14.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:32:14.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:32:14.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:32:14.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:43983
01:32:14.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:32:14.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:32:14.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:32:14.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:32:14.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:45973,127.1.19.189:43805,127.1.19.188:46459
01:32:14.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:32:14.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38635/jwks.json
01:32:14.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:32:14.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.160187 1278 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.189" port: 43805 } has no permanent_uuid. Determining permanent_uuid...
01:32:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:14 dist-test-slave-vbz4 krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790645533, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.189@KRBTEST.COM for kudu/127.1.19.189@KRBTEST.COM
01:32:14.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.195029 1278 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } has no permanent_uuid. Determining permanent_uuid...
01:32:14.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.197619 1227 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.188:46459: connect: Connection refused (error 111)
01:32:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.202548 1278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:46459: Network error: Client connection negotiation failed: client connection to 127.1.19.188:46459: connect: Connection refused (error 111)
01:32:14.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.242606 1197 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.189" port: 43805 } attempt: 7
01:32:14.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.252161 1278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } attempt: 1
01:32:14.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.258316 1278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:46459: Network error: Client connection negotiation failed: client connection to 127.1.19.188:46459: connect: Connection refused (error 111)
01:32:14.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:14 dist-test-slave-vbz4 krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790645531, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.190@KRBTEST.COM for kudu/127.1.19.189@KRBTEST.COM
01:32:14.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.282474 1197 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } has no permanent_uuid. Determining permanent_uuid...
01:32:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.318753 1197 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:46459: Network error: Client connection negotiation failed: client connection to 127.1.19.188:46459: connect: Connection refused (error 111)
01:32:14.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.327021 1278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } attempt: 2
01:32:14.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.334959 1278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:46459: Network error: Client connection negotiation failed: client connection to 127.1.19.188:46459: connect: Connection refused (error 111)
01:32:14.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.377270 1197 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } attempt: 1
01:32:14.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.382022 1197 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:46459: Network error: Client connection negotiation failed: client connection to 127.1.19.188:46459: connect: Connection refused (error 111)
01:32:14.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.426815 1278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } attempt: 3
01:32:14.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.434216 1278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:46459: Network error: Client connection negotiation failed: client connection to 127.1.19.188:46459: connect: Connection refused (error 111)
01:32:14.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.463773 1197 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } attempt: 2
01:32:14.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.469252 1197 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:46459: Network error: Client connection negotiation failed: client connection to 127.1.19.188:46459: connect: Connection refused (error 111)
01:32:14.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.554687 1197 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } attempt: 3
01:32:14.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.559713 1197 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:46459: Network error: Client connection negotiation failed: client connection to 127.1.19.188:46459: connect: Connection refused (error 111)
01:32:14.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.577735 1278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } attempt: 4
01:32:14.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.582273 1278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:46459: Network error: Client connection negotiation failed: client connection to 127.1.19.188:46459: connect: Connection refused (error 111)
01:32:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.616747 1285 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:32:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.617465 1285 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:32:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.618122 1285 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:32:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.655980 1285 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:32:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.656253 1285 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:32:14.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.656548 1285 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:32:14.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.657034 1285 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:32:14.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.657342 1285 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:32:14.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.657564 1285 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:32:14.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.657733 1285 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38635/jwks.json
01:32:14.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.700201 1197 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } attempt: 4
01:32:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.701638 1285 master_runner.cc:386] Master server non-default flags:
01:32:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:32:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:43983
01:32:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:32:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:32:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/data
01:32:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/wal
01:32:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:32:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:32:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:45973,127.1.19.189:43805,127.1.19.188:46459
01:32:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kudu_127.1.19.188.keytab
01:32:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.188
01:32:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:32:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:32:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:32:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.188:46459
01:32:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:32:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:32:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:32:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38635/jwks.json
01:32:14.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:32:14.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:32:14.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:32:14.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:32:14.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:32:14.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/data/info.pb
01:32:14.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.188
01:32:14.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
01:32:14.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:32:14.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:32:14.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:32:14.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:32:14.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:32:14.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/logs
01:32:14.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:32:14.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:32:14.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:32:14.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:32:14.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:32:14.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:32:14.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:32:14.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:32:14.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:32:14.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:32:14.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.703558 1285 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:32:14.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.704948 1285 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:32:14.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.705385 1197 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:46459: Network error: Client connection negotiation failed: client connection to 127.1.19.188:46459: connect: Connection refused (error 111)
01:32:14.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.732726 1295 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:14.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.881971 1278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } attempt: 5
01:32:14.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.886047 1278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:46459: Network error: Client connection negotiation failed: client connection to 127.1.19.188:46459: connect: Connection refused (error 111)
01:32:14.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:14.989001 1197 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } attempt: 5
01:32:14.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.993789 1197 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:46459: Network error: Client connection negotiation failed: client connection to 127.1.19.188:46459: connect: Connection refused (error 111)
01:32:15.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:15.433634 1278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } attempt: 6
01:32:15.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:15.467587 1278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:46459: Network error: Client connection negotiation failed: client connection to 127.1.19.188:46459: connect: Connection refused (error 111)
01:32:15.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:15.546346 1197 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } attempt: 6
01:32:15.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:15.553763 1197 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:46459: Network error: Client connection negotiation failed: client connection to 127.1.19.188:46459: connect: Connection refused (error 111)
01:32:16.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.735733 1296 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:16.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:14.733664 1298 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:16.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:16.481899 1297 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1746 milliseconds
01:32:16.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.482010 1285 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:32:16.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.483546 1285 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:32:16.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.487254 1285 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:32:16.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.488793 1285 hybrid_clock.cc:648] HybridClock initialized: now 1790645536488704 us; error 109 us; skew 500 ppm
01:32:16.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.489737 1285 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:32:16.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:16 dist-test-slave-vbz4 krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790645536, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:32:16.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.496970 1285 init.cc:377] Logged in from keytab as kudu/127.1.19.188@KRBTEST.COM (short username kudu)
01:32:16.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.505751 1285 webserver.cc:460] Webserver started at http://127.1.19.188:41761/ using document root <none> and password file <none>
01:32:16.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.506953 1285 fs_manager.cc:359] Metadata directory not provided
01:32:16.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.507181 1285 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:32:16.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.507776 1285 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
01:32:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.513072 1285 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/data/instance:
01:32:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "94d93b0545d540f4873543e5f028b469"
01:32:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:32:16 on dist-test-slave-vbz4"
01:32:16.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.514269 1285 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/wal/instance:
01:32:16.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "94d93b0545d540f4873543e5f028b469"
01:32:16.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:32:16 on dist-test-slave-vbz4"
01:32:16.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.525276 1285 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.010s sys 0.001s
01:32:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.528056 1278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } attempt: 7
01:32:16.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.535609 1313 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:32:16.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:16.536827 1278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:46459: Network error: Client connection negotiation failed: client connection to 127.1.19.188:46459: connect: Connection refused (error 111)
01:32:16.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.537312 1285 fs_manager.cc:725] Time spent opening block manager: real 0.007s user 0.003s sys 0.005s
01:32:16.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.538004 1285 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/data,/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/wal
01:32:16.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "94d93b0545d540f4873543e5f028b469"
01:32:16.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:32:16 on dist-test-slave-vbz4"
01:32:16.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.538509 1285 fs_report.cc:383] FS layout report
01:32:16.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:32:16.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/wal
01:32:16.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/wal
01:32:16.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/data/data
01:32:16.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:32:16.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:32:16.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:32:16.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:32:16.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:32:16.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:32:16.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:32:16.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:32:16.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:32:16.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:32:16.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:32:16.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.581569 1285 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:32:16.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.587284 1197 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } attempt: 7
01:32:16.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.590294 1285 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:32:16.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.591653 1285 kserver.cc:163] Server-wide thread pool size limit: 3276
01:32:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:16.594386 1197 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:46459: Network error: Client connection negotiation failed: client connection to 127.1.19.188:46459: connect: Connection refused (error 111)
01:32:16.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.700898 1285 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.188:46459
01:32:16.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.700920 1367 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.188:46459 every 8 connection(s)
01:32:16.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.703861 1285 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/data/info.pb
01:32:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.710651 1368 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
01:32:16.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.711858 1102 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu as pid 1285
01:32:16.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.712616 1102 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/wal/instance
01:32:16.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.728363 1368 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.190" port: 45973 } has no permanent_uuid. Determining permanent_uuid...
01:32:16.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:16 dist-test-slave-vbz4 krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790645536, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.188@KRBTEST.COM for kudu/127.1.19.190@KRBTEST.COM
01:32:16.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:16 dist-test-slave-vbz4 krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790645529, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.188@KRBTEST.COM
01:32:16.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.783129 1368 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.189" port: 43805 } has no permanent_uuid. Determining permanent_uuid...
01:32:16.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:16 dist-test-slave-vbz4 krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790645536, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.188@KRBTEST.COM for kudu/127.1.19.189@KRBTEST.COM
01:32:16.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:16 dist-test-slave-vbz4 krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790645529, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.189@KRBTEST.COM
01:32:16.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.818137 1368 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } has no permanent_uuid. Determining permanent_uuid...
01:32:16.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:16 dist-test-slave-vbz4 krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790645536, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.188@KRBTEST.COM for kudu/127.1.19.188@KRBTEST.COM
01:32:16.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:16 dist-test-slave-vbz4 krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790645529, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.190@KRBTEST.COM
01:32:16.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.854223 1102 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
01:32:16.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.854353 1368 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: Bootstrap starting.
01:32:16.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.862862 1368 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: Neither blocks nor log segments found. Creating new log.
01:32:16.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.865548 1368 log.cc:824] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: Log is configured to *not* fsync() on all Append() calls
01:32:16.866 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:32:16.868 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
01:32:16.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.873045 1368 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: No bootstrap required, opened a new log
01:32:16.891 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
01:32:16.894 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
01:32:16.901 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.19.190"
port: 45973
}
bound_http_address {
host: "127.1.19.190"
port: 39973
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.19.189"
port: 43805
}
bound_http_address {
host: "127.1.19.189"
port: 45633
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.19.188"
port: 46459
}
bound_http_address {
host: "127.1.19.188"
port: 41761
}
}
}
01:32:16.901 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
01:32:16.901 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:32:16.901 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
01:32:16.903 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
01:32:16.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.910852 1368 raft_consensus.cc:348] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea15bad27d404c6da448f64733d59bea" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 45973 } } peers { permanent_uuid: "3184c3d2e2de4b37ae9a8742f74bd860" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 43805 } } peers { permanent_uuid: "94d93b0545d540f4873543e5f028b469" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } }
01:32:16.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.911587 1368 raft_consensus.cc:374] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:32:16.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.911821 1368 raft_consensus.cc:729] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 94d93b0545d540f4873543e5f028b469, State: Initialized, Role: FOLLOWER
01:32:16.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.912663 1368 consensus_queue.cc:260] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea15bad27d404c6da448f64733d59bea" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 45973 } } peers { permanent_uuid: "3184c3d2e2de4b37ae9a8742f74bd860" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 43805 } } peers { permanent_uuid: "94d93b0545d540f4873543e5f028b469" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } }
01:32:16.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.916706 1379 sys_catalog.cc:455] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea15bad27d404c6da448f64733d59bea" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 45973 } } peers { permanent_uuid: "3184c3d2e2de4b37ae9a8742f74bd860" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 43805 } } peers { permanent_uuid: "94d93b0545d540f4873543e5f028b469" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } } }
01:32:16.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.917678 1379 sys_catalog.cc:458] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [sys.catalog]: This master's current role is: FOLLOWER
01:32:16.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.918676 1368 sys_catalog.cc:564] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [sys.catalog]: configured and running, proceeding with master startup.
01:32:16.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:16 dist-test-slave-vbz4 krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790645536, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:32:16.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:32:16.944 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:32:16.947 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/krb5cc
01:32:16.957 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:32:16.959 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/krb5cc
01:32:16.961 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:32:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:16.967589 1391 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:16.967967 1391 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:16.968 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
01:32:16.971 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
01:32:16.971 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
01:32:16.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.952503 1285 master_runner.cc:186] Error getting master registration for 127.1.19.190:45973: OK, instance_id { permanent_uuid: "ea15bad27d404c6da448f64733d59bea" instance_seqno: 1790645531834573 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
01:32:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:16.987486 1285 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:32:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:17 dist-test-slave-vbz4 krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790645537, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:32:17.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
01:32:17.010 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:32:17.011 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/krb5cc
01:32:17.012 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
01:32:17.013 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.189:43805
01:32:17.014 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
01:32:17.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:17.015271 1102 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu with pid 1203
01:32:17.039 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:32:17.039 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.188:46459
01:32:17.039 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
01:32:17.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:17.040270 1102 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu with pid 1285
01:32:17.061 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:32:17.062 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.190:45973
01:32:17.062 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
01:32:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:17.062913 1102 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu with pid 1128
01:32:17.085 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:32:17.086 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.189:43805
01:32:17.088 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
01:32:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:17.089798 1102 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:32:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:32:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/wal
01:32:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/data
01:32:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:32:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:32:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:32:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:32:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:32:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:32:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/logs
01:32:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/data/info.pb
01:32:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:32:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:32:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:32:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:32:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:32:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:32:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:32:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:32:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:32:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:32:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:32:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:32:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.189:43805
01:32:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.189
01:32:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45633
01:32:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kudu_127.1.19.189.keytab
01:32:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.189
01:32:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:32:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:32:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:32:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:43983
01:32:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:32:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:32:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:32:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:32:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:45973,127.1.19.189:43805,127.1.19.188:46459
01:32:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:32:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38635/jwks.json
01:32:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:32:17.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:17.450444 1395 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:32:17.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:17.451287 1395 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:32:17.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:17.452423 1395 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:32:17.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:17.493871 1395 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:32:17.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:17.494115 1395 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:32:17.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:17.494388 1395 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:32:17.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:17.494573 1395 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:32:17.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:17.494748 1395 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:32:17.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:17.494932 1395 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:32:17.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:17.495061 1395 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38635/jwks.json
01:32:17.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:17.539355 1395 master_runner.cc:386] Master server non-default flags:
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:43983
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/data
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/wal
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:45973,127.1.19.189:43805,127.1.19.188:46459
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kudu_127.1.19.189.keytab
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.189
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.189:43805
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38635/jwks.json
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/data/info.pb
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.189
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45633
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:32:17.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:32:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/logs
01:32:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:32:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:32:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:32:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:32:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:32:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:32:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:32:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:32:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:32:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:32:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:17.540715 1395 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:32:17.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:17.541713 1395 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:32:17.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:17.561939 1402 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:18.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:17.561939 1401 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:18.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:17.564214 1395 server_base.cc:1034] running on GCE node
01:32:18.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:17.561944 1404 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:18.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:18.907644 1395 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:32:18.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:18.911239 1395 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:32:18.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:18.912919 1395 hybrid_clock.cc:648] HybridClock initialized: now 1790645538912820 us; error 84 us; skew 500 ppm
01:32:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:18.914119 1395 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:32:18.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:18 dist-test-slave-vbz4 krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790645538, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:32:18.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:18.929219 1395 init.cc:377] Logged in from keytab as kudu/127.1.19.189@KRBTEST.COM (short username kudu)
01:32:18.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:18.937247 1395 webserver.cc:460] Webserver started at http://127.1.19.189:45633/ using document root <none> and password file <none>
01:32:18.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:18.939241 1395 fs_manager.cc:359] Metadata directory not provided
01:32:18.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:18.939627 1395 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:32:18.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:18.949877 1395 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.001s sys 0.004s
01:32:18.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:18.956686 1413 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:32:18.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:18.958330 1395 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.001s sys 0.003s
01:32:18.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:18.958750 1395 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/data,/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/wal
01:32:18.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3184c3d2e2de4b37ae9a8742f74bd860"
01:32:18.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:32:13 on dist-test-slave-vbz4"
01:32:18.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:18.961388 1395 fs_report.cc:383] FS layout report
01:32:18.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:32:18.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/wal
01:32:18.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/wal
01:32:18.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/data/data
01:32:18.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:32:18.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:32:18.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:32:18.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:32:18.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:32:18.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:32:18.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:32:18.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:32:18.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:32:18.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:32:18.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:32:19.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:19.020512 1395 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:32:19.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:19.032377 1395 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:32:19.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:19.033015 1395 kserver.cc:163] Server-wide thread pool size limit: 3276
01:32:19.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:19.131510 1395 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.189:43805
01:32:19.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:19.131866 1464 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.189:43805 every 8 connection(s)
01:32:19.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:19.136775 1395 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/data/info.pb
01:32:19.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:19.150146 1465 sys_catalog.cc:263] Verifying existing consensus state
01:32:19.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 01:32:19.151041 1465 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:32:19.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:19.151553 1395 master.cc:537] Master@127.1.19.189:43805 shutting down...
01:32:19.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:19.184880 1102 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu as pid 1395
01:32:19.186 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:32:19.186 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.188:46459
01:32:19.186 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
01:32:19.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:19.188393 1102 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:32:19.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:32:19.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/wal
01:32:19.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/data
01:32:19.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:32:19.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:32:19.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:32:19.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:32:19.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:32:19.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:32:19.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/logs
01:32:19.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/data/info.pb
01:32:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:32:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:32:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:32:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:32:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:32:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:32:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:32:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:32:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:32:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:32:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:32:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:32:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.188:46459
01:32:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.188
01:32:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41761
01:32:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kudu_127.1.19.188.keytab
01:32:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.188
01:32:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:32:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:32:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:32:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:43983
01:32:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:32:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:32:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:32:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:32:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:45973,127.1.19.189:43805,127.1.19.188:46459
01:32:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:32:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38635/jwks.json
01:32:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:32:19.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:19.223910 1395 master.cc:552] Master@127.1.19.189:43805 shutdown complete.
01:32:19.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:32:19.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:19.566618 1467 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:32:19.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:19.567540 1467 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:32:19.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:19.568482 1467 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:32:19.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:19.601917 1467 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:32:19.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:19.602257 1467 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:32:19.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:19.602545 1467 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:32:19.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:19.602748 1467 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:32:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:19.602926 1467 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:32:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:19.603111 1467 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:32:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:19.603259 1467 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38635/jwks.json
01:32:19.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:19.643611 1467 master_runner.cc:386] Master server non-default flags:
01:32:19.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:32:19.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:43983
01:32:19.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:32:19.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:32:19.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/data
01:32:19.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/wal
01:32:19.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:32:19.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:32:19.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:45973,127.1.19.189:43805,127.1.19.188:46459
01:32:19.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kudu_127.1.19.188.keytab
01:32:19.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.188
01:32:19.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:32:19.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:32:19.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:32:19.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.188:46459
01:32:19.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:32:19.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:32:19.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:32:19.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38635/jwks.json
01:32:19.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:32:19.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:32:19.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:32:19.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:32:19.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:32:19.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/data/info.pb
01:32:19.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.188
01:32:19.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41761
01:32:19.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:32:19.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:32:19.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:32:19.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:32:19.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:32:19.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/logs
01:32:19.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:32:19.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:32:19.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:32:19.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:32:19.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:32:19.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:32:19.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:32:19.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:32:19.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:32:19.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:32:19.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:19.645078 1467 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:32:19.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:19.646508 1467 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:32:19.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:19.668908 1476 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:21.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:19.668881 1473 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:21.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:19.668893 1474 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:21.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:19.670804 1467 server_base.cc:1034] running on GCE node
01:32:21.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.016875 1467 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:32:21.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.020280 1467 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:32:21.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.021821 1467 hybrid_clock.cc:648] HybridClock initialized: now 1790645541021771 us; error 65 us; skew 500 ppm
01:32:21.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.022742 1467 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:32:21.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:21 dist-test-slave-vbz4 krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790645541, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:32:21.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.030129 1467 init.cc:377] Logged in from keytab as kudu/127.1.19.188@KRBTEST.COM (short username kudu)
01:32:21.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.036321 1467 webserver.cc:460] Webserver started at http://127.1.19.188:41761/ using document root <none> and password file <none>
01:32:21.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.037528 1467 fs_manager.cc:359] Metadata directory not provided
01:32:21.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.037844 1467 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:32:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.047811 1467 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.005s sys 0.002s
01:32:21.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.052632 1485 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:32:21.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.053840 1467 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
01:32:21.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.054248 1467 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/data,/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/wal
01:32:21.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "94d93b0545d540f4873543e5f028b469"
01:32:21.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:32:16 on dist-test-slave-vbz4"
01:32:21.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.056770 1467 fs_report.cc:383] FS layout report
01:32:21.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:32:21.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/wal
01:32:21.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/wal
01:32:21.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/data/data
01:32:21.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:32:21.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:32:21.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:32:21.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:32:21.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:32:21.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:32:21.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:32:21.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:32:21.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:32:21.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:32:21.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:32:21.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.101212 1467 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:32:21.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.107264 1467 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:32:21.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.108031 1467 kserver.cc:163] Server-wide thread pool size limit: 3276
01:32:21.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.193790 1467 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.188:46459
01:32:21.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.193912 1536 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.188:46459 every 8 connection(s)
01:32:21.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.196676 1467 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-2/data/info.pb
01:32:21.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.203783 1102 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu as pid 1467
01:32:21.205 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:32:21.205 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.190:45973
01:32:21.205 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
01:32:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.206849 1102 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:32:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu
01:32:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/wal
01:32:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/data
01:32:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
01:32:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
01:32:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
01:32:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:32:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:32:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:32:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/logs
01:32:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/data/info.pb
01:32:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:32:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
01:32:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
01:32:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
01:32:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
01:32:21.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:32:21.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:32:21.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:32:21.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
01:32:21.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
01:32:21.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:32:21.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:32:21.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.190:45973
01:32:21.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.190
01:32:21.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39973
01:32:21.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kudu_127.1.19.190.keytab
01:32:21.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.190
01:32:21.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:32:21.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
01:32:21.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
01:32:21.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:43983
01:32:21.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:32:21.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:32:21.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:32:21.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:32:21.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:45973,127.1.19.189:43805,127.1.19.188:46459
01:32:21.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:32:21.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38635/jwks.json
01:32:21.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
01:32:21.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.208096 1537 sys_catalog.cc:263] Verifying existing consensus state
01:32:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.214061 1537 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: Bootstrap starting.
01:32:21.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.256839 1537 log.cc:824] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: Log is configured to *not* fsync() on all Append() calls
01:32:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.261443 1537 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
01:32:21.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.262123 1537 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: Bootstrap complete.
01:32:21.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.297444 1537 raft_consensus.cc:348] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea15bad27d404c6da448f64733d59bea" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 45973 } } peers { permanent_uuid: "3184c3d2e2de4b37ae9a8742f74bd860" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 43805 } } peers { permanent_uuid: "94d93b0545d540f4873543e5f028b469" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } }
01:32:21.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.301358 1537 raft_consensus.cc:374] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
01:32:21.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.301757 1537 raft_consensus.cc:729] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 94d93b0545d540f4873543e5f028b469, State: Initialized, Role: FOLLOWER
01:32:21.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.302911 1537 consensus_queue.cc:260] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea15bad27d404c6da448f64733d59bea" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 45973 } } peers { permanent_uuid: "3184c3d2e2de4b37ae9a8742f74bd860" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 43805 } } peers { permanent_uuid: "94d93b0545d540f4873543e5f028b469" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } }
01:32:21.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.306216 1541 sys_catalog.cc:455] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea15bad27d404c6da448f64733d59bea" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 45973 } } peers { permanent_uuid: "3184c3d2e2de4b37ae9a8742f74bd860" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 43805 } } peers { permanent_uuid: "94d93b0545d540f4873543e5f028b469" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } } }
01:32:21.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.307518 1541 sys_catalog.cc:458] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [sys.catalog]: This master's current role is: FOLLOWER
01:32:21.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.308118 1537 sys_catalog.cc:564] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [sys.catalog]: configured and running, proceeding with master startup.
01:32:21.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:21.356669 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:21.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:21.357383 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:21.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:21.363819 1491 tablet.cc:2369] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: Can't schedule compaction. Clean time has not been advanced past its initial value.
01:32:21.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.342337 1467 master_runner.cc:186] Error getting master registration for 127.1.19.190:45973: Network error: Client connection negotiation failed: client connection to 127.1.19.190:45973: connect: Connection refused (error 111),
01:32:21.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.384832 1467 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:32:21.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.407670 1541 raft_consensus.cc:482] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:32:21.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.408330 1541 raft_consensus.cc:504] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea15bad27d404c6da448f64733d59bea" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 45973 } } peers { permanent_uuid: "3184c3d2e2de4b37ae9a8742f74bd860" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 43805 } } peers { permanent_uuid: "94d93b0545d540f4873543e5f028b469" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } }
01:32:21.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:21.413617 1486 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.190:45973: connect: Connection refused (error 111)
01:32:21.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.417122 1541 leader_election.cc:288] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ea15bad27d404c6da448f64733d59bea (127.1.19.190:45973), 3184c3d2e2de4b37ae9a8742f74bd860 (127.1.19.189:43805)
01:32:21.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:21.422722 1486 leader_election.cc:334] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ea15bad27d404c6da448f64733d59bea (127.1.19.190:45973): Network error: Client connection negotiation failed: client connection to 127.1.19.190:45973: connect: Connection refused (error 111)
01:32:21.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:21.427093 1489 leader_election.cc:334] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3184c3d2e2de4b37ae9a8742f74bd860 (127.1.19.189:43805): Network error: Client connection negotiation failed: client connection to 127.1.19.189:43805: connect: Connection refused (error 111)
01:32:21.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.427623 1489 leader_election.cc:302] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 94d93b0545d540f4873543e5f028b469; no voters: 3184c3d2e2de4b37ae9a8742f74bd860, ea15bad27d404c6da448f64733d59bea
01:32:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.428495 1541 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:32:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:21.635707 1539 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
01:32:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:21.636390 1539 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
01:32:21.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:21.637140 1539 flags.cc:425] Enabled unsafe flag: --never_fsync=true
01:32:21.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:21.672286 1539 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
01:32:21.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:21.672565 1539 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
01:32:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:21.672950 1539 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
01:32:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:21.673272 1539 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
01:32:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:21.673581 1539 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
01:32:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:21.673899 1539 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
01:32:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:21.674125 1539 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38635/jwks.json
01:32:21.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.712811 1539 master_runner.cc:386] Master server non-default flags:
01:32:21.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
01:32:21.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:43983
01:32:21.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
01:32:21.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/data
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/wal
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:45973,127.1.19.189:43805,127.1.19.188:46459
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/krb5kdc/kudu_127.1.19.190.keytab
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.190
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.190:45973
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38635/jwks.json
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/data/info.pb
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.190
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39973
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
01:32:21.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/logs
01:32:21.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
01:32:21.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
01:32:21.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
01:32:21.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
01:32:21.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
01:32:21.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
01:32:21.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
01:32:21.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 00:23:04 UTC on ccycloud.quasar-noncbr.root.comops.site
01:32:21.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4678933
01:32:21.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
01:32:21.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.714874 1539 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
01:32:21.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:21.716109 1539 file_cache.cc:493] Constructed file cache file cache with capacity 419430
01:32:21.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:21.737540 1563 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:22.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:22.365348 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:22.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:22.369114 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.141932 1570 raft_consensus.cc:482] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:32:23.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.142447 1570 raft_consensus.cc:504] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea15bad27d404c6da448f64733d59bea" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 45973 } } peers { permanent_uuid: "3184c3d2e2de4b37ae9a8742f74bd860" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 43805 } } peers { permanent_uuid: "94d93b0545d540f4873543e5f028b469" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } }
01:32:23.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.145334 1570 leader_election.cc:288] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ea15bad27d404c6da448f64733d59bea (127.1.19.190:45973), 3184c3d2e2de4b37ae9a8742f74bd860 (127.1.19.189:43805)
01:32:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:21.737673 1562 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:21.738072 1565 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
01:32:23.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:23.147017 1564 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
01:32:23.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.147428 1539 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
01:32:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.153074 1539 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
01:32:23.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:23.163540 1486 leader_election.cc:334] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ea15bad27d404c6da448f64733d59bea (127.1.19.190:45973): Network error: Client connection negotiation failed: client connection to 127.1.19.190:45973: connect: Connection refused (error 111)
01:32:23.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.167651 1539 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
01:32:23.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.169495 1539 hybrid_clock.cc:648] HybridClock initialized: now 1790645543169398 us; error 97 us; skew 500 ppm
01:32:23.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.170925 1539 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
01:32:23.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:23.176080 1489 leader_election.cc:334] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3184c3d2e2de4b37ae9a8742f74bd860 (127.1.19.189:43805): Network error: Client connection negotiation failed: client connection to 127.1.19.189:43805: connect: Connection refused (error 111)
01:32:23.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.176786 1489 leader_election.cc:302] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 94d93b0545d540f4873543e5f028b469; no voters: 3184c3d2e2de4b37ae9a8742f74bd860, ea15bad27d404c6da448f64733d59bea
01:32:23.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.177695 1570 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:32:23.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:23 dist-test-slave-vbz4 krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790645543, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:32:23.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.179522 1539 init.cc:377] Logged in from keytab as kudu/127.1.19.190@KRBTEST.COM (short username kudu)
01:32:23.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.189002 1539 webserver.cc:460] Webserver started at http://127.1.19.190:39973/ using document root <none> and password file <none>
01:32:23.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.190268 1539 fs_manager.cc:359] Metadata directory not provided
01:32:23.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.190570 1539 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
01:32:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.207168 1539 fs_manager.cc:709] Time spent opening directory manager: real 0.012s user 0.013s sys 0.000s
01:32:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.216105 1579 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
01:32:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.217898 1539 fs_manager.cc:725] Time spent opening block manager: real 0.008s user 0.002s sys 0.006s
01:32:23.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.218327 1539 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/data,/tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/wal
01:32:23.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea15bad27d404c6da448f64733d59bea"
01:32:23.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 01:32:11 on dist-test-slave-vbz4"
01:32:23.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.221903 1539 fs_report.cc:383] FS layout report
01:32:23.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
01:32:23.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/wal
01:32:23.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/wal
01:32:23.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/data/data
01:32:23.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
01:32:23.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
01:32:23.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
01:32:23.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
01:32:23.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
01:32:23.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
01:32:23.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
01:32:23.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
01:32:23.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
01:32:23.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
01:32:23.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
01:32:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.293514 1539 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
01:32:23.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.300469 1539 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
01:32:23.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.301008 1539 kserver.cc:163] Server-wide thread pool size limit: 3276
01:32:23.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:23.375212 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:23.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:23.375830 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:23.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.399025 1539 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.190:45973
01:32:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.399149 1630 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.190:45973 every 8 connection(s)
01:32:23.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.402997 1539 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/data/info.pb
01:32:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.409324 1102 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu as pid 1539
01:32:23.410 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
01:32:23.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.418258 1631 sys_catalog.cc:263] Verifying existing consensus state
01:32:23.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 01:32:23.419313 1631 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:32:23.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.419813 1539 master.cc:537] Master@127.1.19.190:45973 shutting down...
01:32:23.485 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.189:43805(127.1.19.189:43805)] connecting to peer
01:32:23.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:23.536208 1539 master.cc:552] Master@127.1.19.190:45973 shutdown complete.
01:32:23.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskCdr2xp/test-tmp/mini-kudu-cluster7812016760943089565/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
01:32:23.589 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.188:46459(127.1.19.188:46459)] connecting to peer
01:32:23.600 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.190:45973(127.1.19.190:45973)] connecting to peer
01:32:23.668 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.1.19.188:46459(127.1.19.188:46459)] Successfully connected to peer
01:32:23.668 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:23.668 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:43805(127.1.19.189:43805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:23.670 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:43805: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:23.672 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:23.672 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.190:45973(127.1.19.190:45973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:23.673 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:45973: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:23.763 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
01:32:23.774 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.1.19.188
01:32:23.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:23.850544 525 debug-util.cc:398] Leaking SignalData structure 0x7b08000bbb00 after lost signal to thread 460
01:32:23.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:23.851265 525 debug-util.cc:398] Leaking SignalData structure 0x7b08000bb6a0 after lost signal to thread 528
01:32:24.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:24 dist-test-slave-vbz4 krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790645537, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.188@KRBTEST.COM
01:32:24.057 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x5b3b15bb, L:/127.0.0.1:60224 - R:/127.1.19.188:46459] using SASL/GSSAPI
01:32:24.076 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973; will retry
01:32:24.096 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:43805(127.1.19.189:43805)] connecting to peer
01:32:24.102 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:45973(127.1.19.190:45973)] connecting to peer
01:32:24.104 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.190:45973(127.1.19.190:45973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.114 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:45973: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.115 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.115 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.189:43805(127.1.19.189:43805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.116 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:43805: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.120 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973; will retry
01:32:24.135 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:43805(127.1.19.189:43805)] connecting to peer
01:32:24.138 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:45973(127.1.19.190:45973)] connecting to peer
01:32:24.141 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.142 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.190:45973(127.1.19.190:45973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.146 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:45973: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.147 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.147 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.189:43805(127.1.19.189:43805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.148 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:43805: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.149 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973; will retry
01:32:24.175 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:43805(127.1.19.189:43805)] connecting to peer
01:32:24.176 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:45973(127.1.19.190:45973)] connecting to peer
01:32:24.180 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.180 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.190:45973(127.1.19.190:45973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.181 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.181 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.189:43805(127.1.19.189:43805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.181 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:45973: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.182 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:43805: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.186 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973; will retry
01:32:24.215 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:43805(127.1.19.189:43805)] connecting to peer
01:32:24.216 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:45973(127.1.19.190:45973)] connecting to peer
01:32:24.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.190:45973(127.1.19.190:45973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.219 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:45973: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.222 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.222 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.189:43805(127.1.19.189:43805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.223 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:43805: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.227 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973; will retry
01:32:24.275 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:43805(127.1.19.189:43805)] connecting to peer
01:32:24.277 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:45973(127.1.19.190:45973)] connecting to peer
01:32:24.280 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.280 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.190:45973(127.1.19.190:45973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.281 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:45973: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.281 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.282 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.189:43805(127.1.19.189:43805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.282 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:43805: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.285 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973; will retry
01:32:24.314 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:43805(127.1.19.189:43805)] connecting to peer
01:32:24.317 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:45973(127.1.19.190:45973)] connecting to peer
01:32:24.319 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.320 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.189:43805(127.1.19.189:43805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.321 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.321 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.190:45973(127.1.19.190:45973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.321 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:43805: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.323 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:45973: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.323 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973; will retry
01:32:24.374 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:43805(127.1.19.189:43805)] connecting to peer
01:32:24.377 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:45973(127.1.19.190:45973)] connecting to peer
01:32:24.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.189:43805(127.1.19.189:43805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.379 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:43805: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.380 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.380 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.190:45973(127.1.19.190:45973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.381 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:45973: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:24.382203 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:24.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:24.382738 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:24.383 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973; will retry
01:32:24.574 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:43805(127.1.19.189:43805)] connecting to peer
01:32:24.576 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.577 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:45973(127.1.19.190:45973)] connecting to peer
01:32:24.577 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.189:43805(127.1.19.189:43805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:43805: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:24.581 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.581 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.190:45973(127.1.19.190:45973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.582 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:45973: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:24.584 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973; will retry
01:32:25.075 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:43805(127.1.19.189:43805)] connecting to peer
01:32:25.078 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:45973(127.1.19.190:45973)] connecting to peer
01:32:25.081 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:25.081 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:25.081 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.190:45973(127.1.19.190:45973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:25.081 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.189:43805(127.1.19.189:43805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:25.083 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:45973: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:25.083 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:43805: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:25.086 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973; will retry
01:32:25.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:25.386183 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:25.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:25.386487 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:25.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:25.496047 605 debug-util.cc:398] Leaking SignalData structure 0x7b080009d6c0 after lost signal to thread 536
01:32:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:25.496686 605 debug-util.cc:398] Leaking SignalData structure 0x7b08000baaa0 after lost signal to thread 608
01:32:25.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:25.768090 1640 raft_consensus.cc:482] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:32:25.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:25.768522 1640 raft_consensus.cc:504] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea15bad27d404c6da448f64733d59bea" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 45973 } } peers { permanent_uuid: "3184c3d2e2de4b37ae9a8742f74bd860" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 43805 } } peers { permanent_uuid: "94d93b0545d540f4873543e5f028b469" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } }
01:32:25.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:25.770247 1640 leader_election.cc:288] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ea15bad27d404c6da448f64733d59bea (127.1.19.190:45973), 3184c3d2e2de4b37ae9a8742f74bd860 (127.1.19.189:43805)
01:32:25.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:25.787065 1489 leader_election.cc:334] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3184c3d2e2de4b37ae9a8742f74bd860 (127.1.19.189:43805): Network error: Client connection negotiation failed: client connection to 127.1.19.189:43805: connect: Connection refused (error 111)
01:32:25.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:25.788659 1486 leader_election.cc:334] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ea15bad27d404c6da448f64733d59bea (127.1.19.190:45973): Network error: Client connection negotiation failed: client connection to 127.1.19.190:45973: connect: Connection refused (error 111)
01:32:25.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:25.789036 1486 leader_election.cc:302] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 94d93b0545d540f4873543e5f028b469; no voters: 3184c3d2e2de4b37ae9a8742f74bd860, ea15bad27d404c6da448f64733d59bea
01:32:25.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:25.789747 1640 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:32:25.894 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:43805(127.1.19.189:43805)] connecting to peer
01:32:25.897 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:45973(127.1.19.190:45973)] connecting to peer
01:32:25.900 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:25.900 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.190:45973(127.1.19.190:45973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:25.902 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:45973: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:25.904 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:25.904 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.189:43805(127.1.19.189:43805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:25.905 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:43805: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:25.906 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973; will retry
01:32:26.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:26.388072 1467 master_runner.cc:186] Error getting master registration for 127.1.19.190:45973: Network error: Client connection negotiation failed: client connection to 127.1.19.190:45973: connect: Connection refused (error 111),
01:32:26.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:26.388468 1467 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:32:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:26.391309 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:26.391762 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:26.394 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:43805(127.1.19.189:43805)] connecting to peer
01:32:26.396 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:45973(127.1.19.190:45973)] connecting to peer
01:32:26.397 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:26.397 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.189:43805(127.1.19.189:43805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:26.397 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:43805: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:26.399 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:26.399 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.190:45973(127.1.19.190:45973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:26.400 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:45973: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:26.400 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973; will retry
01:32:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:27.026930 525 debug-util.cc:398] Leaking SignalData structure 0x7b08000bdb40 after lost signal to thread 460
01:32:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:27.027431 525 debug-util.cc:398] Leaking SignalData structure 0x7b08000bfc00 after lost signal to thread 528
01:32:27.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:27.395016 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:27.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:27.395295 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:27.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:27.852926 1646 raft_consensus.cc:482] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:32:27.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:27.853389 1646 raft_consensus.cc:504] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea15bad27d404c6da448f64733d59bea" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 45973 } } peers { permanent_uuid: "3184c3d2e2de4b37ae9a8742f74bd860" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 43805 } } peers { permanent_uuid: "94d93b0545d540f4873543e5f028b469" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } }
01:32:27.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:27.855727 1646 leader_election.cc:288] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ea15bad27d404c6da448f64733d59bea (127.1.19.190:45973), 3184c3d2e2de4b37ae9a8742f74bd860 (127.1.19.189:43805)
01:32:27.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:27.857853 1486 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.190:45973: connect: Connection refused (error 111) [suppressed 5 similar messages]
01:32:27.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:27.861284 1489 leader_election.cc:334] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3184c3d2e2de4b37ae9a8742f74bd860 (127.1.19.189:43805): Network error: Client connection negotiation failed: client connection to 127.1.19.189:43805: connect: Connection refused (error 111)
01:32:27.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:27.861894 1486 leader_election.cc:334] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ea15bad27d404c6da448f64733d59bea (127.1.19.190:45973): Network error: Client connection negotiation failed: client connection to 127.1.19.190:45973: connect: Connection refused (error 111)
01:32:27.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:27.862488 1486 leader_election.cc:302] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 94d93b0545d540f4873543e5f028b469; no voters: 3184c3d2e2de4b37ae9a8742f74bd860, ea15bad27d404c6da448f64733d59bea
01:32:27.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:27.863173 1646 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:32:28.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:28.398887 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:28.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:28.399221 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:29.402082 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:29.402351 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:30.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:30.405970 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:30.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:30.406301 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:30.454 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:43805(127.1.19.189:43805)] connecting to peer
01:32:30.456 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:45973(127.1.19.190:45973)] connecting to peer
01:32:30.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:30.459 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.190:45973(127.1.19.190:45973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:30.459 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:45973: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:30.460 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:30.460 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.189:43805(127.1.19.189:43805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:30.460 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:43805: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:30.462 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973; will retry
01:32:31.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:31 dist-test-slave-vbz4 krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790645551, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:32:31.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:31.035094 1481 init.cc:295] Successfully reacquired a new kerberos TGT
01:32:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:31.391993 1467 master_runner.cc:186] Error getting master registration for 127.1.19.190:45973: Network error: Client connection negotiation failed: client connection to 127.1.19.190:45973: connect: Connection refused (error 111),
01:32:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:31.392385 1467 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:32:31.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:31.409386 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:31.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:31.409662 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:32.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:32.412904 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:32.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:32.413156 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:32.774 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:43805(127.1.19.189:43805)] connecting to peer
01:32:32.776 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:45973(127.1.19.190:45973)] connecting to peer
01:32:32.778 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:32.778 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:32.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.189:43805(127.1.19.189:43805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:32.778 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.190:45973(127.1.19.190:45973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:32.778 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:45973: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:32.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:43805: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:32.781 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973; will retry
01:32:33.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:33.415900 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:33.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:33.416149 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:34.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:34.155646 1657 raft_consensus.cc:482] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:32:34.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:34.156116 1657 raft_consensus.cc:504] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea15bad27d404c6da448f64733d59bea" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 45973 } } peers { permanent_uuid: "3184c3d2e2de4b37ae9a8742f74bd860" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 43805 } } peers { permanent_uuid: "94d93b0545d540f4873543e5f028b469" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } }
01:32:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:34.157841 1657 leader_election.cc:288] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ea15bad27d404c6da448f64733d59bea (127.1.19.190:45973), 3184c3d2e2de4b37ae9a8742f74bd860 (127.1.19.189:43805)
01:32:34.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:34.159893 1486 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.190:45973: connect: Connection refused (error 111) [suppressed 1 similar messages]
01:32:34.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:34.162703 1486 leader_election.cc:334] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ea15bad27d404c6da448f64733d59bea (127.1.19.190:45973): Network error: Client connection negotiation failed: client connection to 127.1.19.190:45973: connect: Connection refused (error 111)
01:32:34.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:34.163192 1489 leader_election.cc:334] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3184c3d2e2de4b37ae9a8742f74bd860 (127.1.19.189:43805): Network error: Client connection negotiation failed: client connection to 127.1.19.189:43805: connect: Connection refused (error 111)
01:32:34.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:34.163558 1489 leader_election.cc:302] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 94d93b0545d540f4873543e5f028b469; no voters: 3184c3d2e2de4b37ae9a8742f74bd860, ea15bad27d404c6da448f64733d59bea
01:32:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:34.164472 1657 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:32:34.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:34.419128 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:34.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:34.419484 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:35.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:35.422487 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:35.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:35.422734 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:36.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:36.395031 1467 master_runner.cc:186] Error getting master registration for 127.1.19.190:45973: Network error: Client connection negotiation failed: client connection to 127.1.19.190:45973: connect: Connection refused (error 111),
01:32:36.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:36.395356 1467 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:32:36.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:36.426265 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:36.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:36.426551 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:36.814 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:43805(127.1.19.189:43805)] connecting to peer
01:32:36.816 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:45973(127.1.19.190:45973)] connecting to peer
01:32:36.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:36.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:36.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.190:45973(127.1.19.190:45973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:36.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.189:43805(127.1.19.189:43805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:36.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:45973: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:36.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:43805: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:36.820 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973; will retry
01:32:37.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:37.429716 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:37.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:37.430052 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:38.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:38.433223 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:38.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:38.433501 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:39.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:39.111469 1087 debug-util.cc:398] Leaking SignalData structure 0x7b08000bde40 after lost signal to thread 962
01:32:39.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:39.112682 1087 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba6a0 after lost signal to thread 1090
01:32:39.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:39.436600 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:39.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:39.436928 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:39.895 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:43805(127.1.19.189:43805)] connecting to peer
01:32:39.897 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:45973(127.1.19.190:45973)] connecting to peer
01:32:39.899 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:39.900 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.190:45973(127.1.19.190:45973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:39.900 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:45973: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:39.901 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:39.901 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.189:43805(127.1.19.189:43805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:39.901 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:43805: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:39.903 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973; will retry
01:32:40.034 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:43805(127.1.19.189:43805)] connecting to peer
01:32:40.035 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:45973(127.1.19.190:45973)] connecting to peer
01:32:40.037 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:40.037 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.190:45973(127.1.19.190:45973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:40.038 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:45973: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:40.040 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:40.040 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.189:43805(127.1.19.189:43805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:40.040 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:43805: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:40.044 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973; will retry
01:32:40.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:40.441176 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:40.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:40.441581 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:41.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 01:32:41 dist-test-slave-vbz4 krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790645561, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
01:32:41.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:41.037931 1481 init.cc:295] Successfully reacquired a new kerberos TGT
01:32:41.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:41.398587 1467 master_runner.cc:186] Error getting master registration for 127.1.19.190:45973: Network error: Client connection negotiation failed: client connection to 127.1.19.190:45973: connect: Connection refused (error 111),
01:32:41.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:41.398955 1467 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:32:41.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:41.444850 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:41.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:41.445494 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:42.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:42.336247 1670 raft_consensus.cc:482] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
01:32:42.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:42.336688 1670 raft_consensus.cc:504] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ea15bad27d404c6da448f64733d59bea" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 45973 } } peers { permanent_uuid: "3184c3d2e2de4b37ae9a8742f74bd860" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 43805 } } peers { permanent_uuid: "94d93b0545d540f4873543e5f028b469" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 46459 } }
01:32:42.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:42.338696 1670 leader_election.cc:288] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ea15bad27d404c6da448f64733d59bea (127.1.19.190:45973), 3184c3d2e2de4b37ae9a8742f74bd860 (127.1.19.189:43805)
01:32:42.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:42.340888 1486 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.190:45973: connect: Connection refused (error 111) [suppressed 1 similar messages]
01:32:42.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:42.344014 1489 leader_election.cc:334] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 3184c3d2e2de4b37ae9a8742f74bd860 (127.1.19.189:43805): Network error: Client connection negotiation failed: client connection to 127.1.19.189:43805: connect: Connection refused (error 111)
01:32:42.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:42.344453 1486 leader_election.cc:334] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ea15bad27d404c6da448f64733d59bea (127.1.19.190:45973): Network error: Client connection negotiation failed: client connection to 127.1.19.190:45973: connect: Connection refused (error 111)
01:32:42.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:42.344753 1486 leader_election.cc:302] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 94d93b0545d540f4873543e5f028b469; no voters: 3184c3d2e2de4b37ae9a8742f74bd860, ea15bad27d404c6da448f64733d59bea
01:32:42.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:42.345355 1670 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
01:32:42.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:42.448992 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:42.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:42.449280 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:42.815 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:43805(127.1.19.189:43805)] connecting to peer
01:32:42.817 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:45973(127.1.19.190:45973)] connecting to peer
01:32:42.819 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:42.819 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:42.819 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.190:45973(127.1.19.190:45973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:42.819 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.189:43805(127.1.19.189:43805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:42.819 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:43805: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:42.819 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:45973: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:42.821 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973; will retry
01:32:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:43.452526 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:43.452816 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:44.456139 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:44.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:44.456427 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:45.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:45.459793 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:45.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:45.460086 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:45.974 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:43805(127.1.19.189:43805)] connecting to peer
01:32:45.976 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:45973(127.1.19.190:45973)] connecting to peer
01:32:45.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:45.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.189:43805(127.1.19.189:43805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:45.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:43805: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:45.979 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:45.979 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.190:45973(127.1.19.190:45973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:45.979 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:45973: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:45.980 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973; will retry
01:32:46.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:46.401568 1467 master_runner.cc:186] Error getting master registration for 127.1.19.190:45973: Network error: Client connection negotiation failed: client connection to 127.1.19.190:45973: connect: Connection refused (error 111),
01:32:46.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:46.401934 1467 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
01:32:46.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:46.464037 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:46.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:46.464388 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:47.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:47.468309 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:47.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:47.468633 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:48.472414 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:48.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:48.472743 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:48.474 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:43805(127.1.19.189:43805)] connecting to peer
01:32:48.476 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:45973(127.1.19.190:45973)] connecting to peer
01:32:48.477 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:48.477 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.190:45973(127.1.19.190:45973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:48.477 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:45973: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:48.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:48.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.189:43805(127.1.19.189:43805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:48.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:43805: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:48.480 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973; will retry
01:32:49.054 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:43805(127.1.19.189:43805)] connecting to peer
01:32:49.055 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:45973(127.1.19.190:45973)] connecting to peer
01:32:49.056 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:49.056 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.189:43805(127.1.19.189:43805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:49.057 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:43805: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:49.057 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:49.057 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.190:45973(127.1.19.190:45973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:49.057 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:45973: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:49.059 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973; will retry
01:32:49.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:49.476754 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:49.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:49.478940 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:50.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:50.482905 1552 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 94d93b0545d540f4873543e5f028b469: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
01:32:50.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:50.483239 1552 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
01:32:50.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:50.496726 1533 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8800 after lost signal to thread 1468
01:32:50.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 01:32:50.497574 1533 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5900 after lost signal to thread 1536
01:32:50.835 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:43805(127.1.19.189:43805)] connecting to peer
01:32:50.837 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:50.837 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.189:43805(127.1.19.189:43805)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:50.837 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:43805: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805
01:32:50.838 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:45973(127.1.19.190:45973)] connecting to peer
01:32:50.839 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:50.839 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.190:45973(127.1.19.190:45973)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:50.839 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:45973: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
01:32:50.840 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973; will retry
01:32:50.843 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=27427), Traces: [0ms] refreshing cache from master, [56ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [171ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [183ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [252ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [255ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [655ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [660ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [678ms] refreshing cache from master, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [685ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [688ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [692ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [698ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [703ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [717ms] refreshing cache from master, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [725ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [729ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [731ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [732ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [765ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [768ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [770ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [797ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [801ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [805ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [807ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [810ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [857ms] refreshing cache from master, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [863ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [865ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [868ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [869ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [897ms] refreshing cache from master, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [903ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [904ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [905ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [906ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [957ms] refreshing cache from master, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [962ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [964ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [965ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [966ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [1157ms] refreshing cache from master, [1157ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [1159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [1160ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [1164ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [1167ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [1168ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [1657ms] refreshing cache from master, [1658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [1659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [1661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [1664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [1664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [1668ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [1669ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [2477ms] refreshing cache from master, [2477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [2479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [2479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [2483ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [2487ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [2489ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [2489ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [2977ms] refreshing cache from master, [2977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [2978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [2979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [2980ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [2982ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [2983ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [2983ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [7037ms] refreshing cache from master, [7037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [7038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [7039ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [7042ms] too many traces: truncated at 100 traces, deferred=Deferred@1725633569(state=PENDING, result=null, callback=(continuation of Deferred@1206941866 after retry RPC after error@353090419) -> (continuation of Deferred@633527108 after retry RPC after error@453425601) -> (continuation of Deferred@1601681187 after retry RPC after error@401563549) -> (continuation of Deferred@616869599 after retry RPC after error@178438967) -> (continuation of Deferred@1745789935 after retry RPC after error@99617476) -> (continuation of Deferred@79953651 after retry RPC after error@409222695) -> (continuation of Deferred@90765087 after retry RPC after error@13335388) -> (continuation of Deferred@467937744 after retry RPC after error@575191437) -> (continuation of Deferred@1624924884 after retry RPC after error@999351304) -> (continuation of Deferred@16834563 after retry RPC after error@1726199167) -> (continuation of Deferred@1436334005 after retry RPC after error@1771092394) -> (continuation of Deferred@1872120931 after retry RPC after error@346375331) -> (continuation of Deferred@615807133 after retry RPC after error@1208338504) -> (continuation of Deferred@355299455 after retry RPC after error@731098661) -> (continuation of Deferred@744770394 after retry RPC after error@1254461718) -> (continuation of Deferred@873840313 after retry RPC after error@37870744) -> (continuation of Deferred@217147665 after retry RPC after error@1792672952) -> (continuation of Deferred@1940102956 after retry RPC after error@2139764956) -> (continuation of Deferred@2025091037 after retry RPC after error@1123109815) -> (continuation of Deferred@759490106 after retry RPC after error@52848534) -> (continuation of Deferred@1488403112 after retry RPC after error@282296058), errback=(continuation of Deferred@1206941866 after retry RPC after error@353090419) -> (continuation of Deferred@633527108 after retry RPC after error@453425601) -> (continuation of Deferred@1601681187 after retry RPC after error@401563549) -> (continuation of Deferred@616869599 after retry RPC after error@178438967) -> (continuation of Deferred@1745789935 after retry RPC after error@99617476) -> (continuation of Deferred@79953651 after retry RPC after error@409222695) -> (continuation of Deferred@90765087 after retry RPC after error@13335388) -> (continuation of Deferred@467937744 after retry RPC after error@575191437) -> (continuation of Deferred@1624924884 after retry RPC after error@999351304) -> (continuation of Deferred@16834563 after retry RPC after error@1726199167) -> (continuation of Deferred@1436334005 after retry RPC after error@1771092394) -> (continuation of Deferred@1872120931 after retry RPC after error@346375331) -> (continuation of Deferred@615807133 after retry RPC after error@1208338504) -> (continuation of Deferred@355299455 after retry RPC after error@731098661) -> (continuation of Deferred@744770394 after retry RPC after error@1254461718) -> (continuation of Deferred@873840313 after retry RPC after error@37870744) -> (continuation of Deferred@217147665 after retry RPC after error@1792672952) -> (continuation of Deferred@1940102956 after retry RPC after error@2139764956) -> (continuation of Deferred@2025091037 after retry RPC after error@1123109815) -> (continuation of Deferred@759490106 after retry RPC after error@52848534) -> (continuation of Deferred@1488403112 after retry RPC after error@282296058)))
01:32:50.848 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.190:45973(127.1.19.190:45973)] disconnecting while in state TERMINATED
01:32:50.849 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.188:46459(127.1.19.188:46459)] disconnecting while in state READY
01:32:50.850 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.189:43805(127.1.19.189:43805)] disconnecting while in state TERMINATED
01:32:50.856 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:32:50.861 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:32:50.862 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.1.19.188:46459(127.1.19.188:46459)] handling channelInactive
01:32:50.862 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.188:46459(127.1.19.188:46459)] cleaning up while in state READY due to: connection closed
01:32:50.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:50.869009 1102 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu with pid 1467
01:32:52.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T01:32:52Z chronyd exiting
01:32:53.089 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
01:32:53.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:53.090704 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu with pid 657
01:32:53.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:53.124490 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu with pid 818
01:32:53.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:53.157527 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu with pid 961
01:32:53.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:53.188520 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu with pid 386
01:32:53.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:53.217983 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu with pid 456
01:32:53.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 01:32:53.256191 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskCdr2xp/build/tsan/bin/kudu with pid 531
01:32:54.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T01:32:54Z chronyd exiting
01:32:55.465 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=27427), Traces: [0ms] refreshing cache from master, [56ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [171ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [183ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [252ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [255ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [655ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [660ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [678ms] refreshing cache from master, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [685ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [688ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [692ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [698ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [703ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [717ms] refreshing cache from master, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [725ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [729ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [731ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [732ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [765ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [768ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [770ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [797ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [801ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [805ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [807ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [810ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [857ms] refreshing cache from master, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [863ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [865ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [868ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [869ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [897ms] refreshing cache from master, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [903ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [904ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [905ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [906ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [957ms] refreshing cache from master, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [962ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [964ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [965ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [966ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [1157ms] refreshing cache from master, [1157ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [1159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [1160ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [1164ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [1167ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [1168ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [1657ms] refreshing cache from master, [1658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [1659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [1661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [1664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [1664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [1668ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [1669ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [2477ms] refreshing cache from master, [2477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [2479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [2479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [2483ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [2487ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [2489ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [2489ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [2977ms] refreshing cache from master, [2977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [2978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [2979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [2980ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [2982ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [2983ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [2983ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [7037ms] refreshing cache from master, [7037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [7038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [7039ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [7042ms] too many traces: truncated at 100 traces, deferred=Deferred@1725633569(state=PENDING, result=null, callback=(continuation of Deferred@1206941866 after retry RPC after error@353090419) -> (continuation of Deferred@633527108 after retry RPC after error@453425601) -> (continuation of Deferred@1601681187 after retry RPC after error@401563549) -> (continuation of Deferred@616869599 after retry RPC after error@178438967) -> (continuation of Deferred@1745789935 after retry RPC after error@99617476) -> (continuation of Deferred@79953651 after retry RPC after error@409222695) -> (continuation of Deferred@90765087 after retry RPC after error@13335388) -> (continuation of Deferred@467937744 after retry RPC after error@575191437) -> (continuation of Deferred@1624924884 after retry RPC after error@999351304) -> (continuation of Deferred@16834563 after retry RPC after error@1726199167) -> (continuation of Deferred@1436334005 after retry RPC after error@1771092394) -> (continuation of Deferred@1872120931 after retry RPC after error@346375331) -> (continuation of Deferred@615807133 after retry RPC after error@1208338504) -> (continuation of Deferred@355299455 after retry RPC after error@731098661) -> (continuation of Deferred@744770394 after retry RPC after error@1254461718) -> (continuation of Deferred@873840313 after retry RPC after error@37870744) -> (continuation of Deferred@217147665 after retry RPC after error@1792672952) -> (continuation of Deferred@1940102956 after retry RPC after error@2139764956) -> (continuation of Deferred@2025091037 after retry RPC after error@1123109815) -> (continuation of Deferred@759490106 after retry RPC after error@52848534) -> (continuation of Deferred@1488403112 after retry RPC after error@282296058), errback=(continuation of Deferred@1206941866 after retry RPC after error@353090419) -> (continuation of Deferred@633527108 after retry RPC after error@453425601) -> (continuation of Deferred@1601681187 after retry RPC after error@401563549) -> (continuation of Deferred@616869599 after retry RPC after error@178438967) -> (continuation of Deferred@1745789935 after retry RPC after error@99617476) -> (continuation of Deferred@79953651 after retry RPC after error@409222695) -> (continuation of Deferred@90765087 after retry RPC after error@13335388) -> (continuation of Deferred@467937744 after retry RPC after error@575191437) -> (continuation of Deferred@1624924884 after retry RPC after error@999351304) -> (continuation of Deferred@16834563 after retry RPC after error@1726199167) -> (continuation of Deferred@1436334005 after retry RPC after error@1771092394) -> (continuation of Deferred@1872120931 after retry RPC after error@346375331) -> (continuation of Deferred@615807133 after retry RPC after error@1208338504) -> (continuation of Deferred@355299455 after retry RPC after error@731098661) -> (continuation of Deferred@744770394 after retry RPC after error@1254461718) -> (continuation of Deferred@873840313 after retry RPC after error@37870744) -> (continuation of Deferred@217147665 after retry RPC after error@1792672952) -> (continuation of Deferred@1940102956 after retry RPC after error@2139764956) -> (continuation of Deferred@2025091037 after retry RPC after error@1123109815) -> (continuation of Deferred@759490106 after retry RPC after error@52848534) -> (continuation of Deferred@1488403112 after retry RPC after error@282296058)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.190:45973
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=27427), Traces: [0ms] refreshing cache from master, [56ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [171ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [183ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [252ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [255ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [655ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [660ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [678ms] refreshing cache from master, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [685ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [688ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [692ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [698ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [703ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [717ms] refreshing cache from master, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [725ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [729ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [731ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [732ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [765ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [768ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [770ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [797ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [801ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [805ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [807ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [810ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [857ms] refreshing cache from master, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [863ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [865ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [868ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [869ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [897ms] refreshing cache from master, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [903ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [904ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [905ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [906ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [957ms] refreshing cache from master, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [962ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [964ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [965ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [966ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [1157ms] refreshing cache from master, [1157ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [1158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [1159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [1160ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [1164ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [1167ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [1168ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [1657ms] refreshing cache from master, [1658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [1659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [1661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [1664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [1664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [1668ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [1669ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [2477ms] refreshing cache from master, [2477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [2479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [2479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [2483ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [2487ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [2489ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [2489ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [2977ms] refreshing cache from master, [2977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [2978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [2979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [2980ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:43805: Network error: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805, [2982ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:45973: Network error: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [2983ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:46459: OK, [2983ms] delaying RPC due to: Service unavailable: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973, [7037ms] refreshing cache from master, [7037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:43805, [7038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:46459, [7039ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:45973, [7042ms] too many traces: truncated at 100 traces, deferred=Deferred@1725633569(state=PENDING, result=null, callback=(continuation of Deferred@1206941866 after retry RPC after error@353090419) -> (continuation of Deferred@633527108 after retry RPC after error@453425601) -> (continuation of Deferred@1601681187 after retry RPC after error@401563549) -> (continuation of Deferred@616869599 after retry RPC after error@178438967) -> (continuation of Deferred@1745789935 after retry RPC after error@99617476) -> (continuation of Deferred@79953651 after retry RPC after error@409222695) -> (continuation of Deferred@90765087 after retry RPC after error@13335388) -> (continuation of Deferred@467937744 after retry RPC after error@575191437) -> (continuation of Deferred@1624924884 after retry RPC after error@999351304) -> (continuation of Deferred@16834563 after retry RPC after error@1726199167) -> (continuation of Deferred@1436334005 after retry RPC after error@1771092394) -> (continuation of Deferred@1872120931 after retry RPC after error@346375331) -> (continuation of Deferred@615807133 after retry RPC after error@1208338504) -> (continuation of Deferred@355299455 after retry RPC after error@731098661) -> (continuation of Deferred@744770394 after retry RPC after error@1254461718) -> (continuation of Deferred@873840313 after retry RPC after error@37870744) -> (continuation of Deferred@217147665 after retry RPC after error@1792672952) -> (continuation of Deferred@1940102956 after retry RPC after error@2139764956) -> (continuation of Deferred@2025091037 after retry RPC after error@1123109815) -> (continuation of Deferred@759490106 after retry RPC after error@52848534) -> (continuation of Deferred@1488403112 after retry RPC after error@282296058), errback=(continuation of Deferred@1206941866 after retry RPC after error@353090419) -> (continuation of Deferred@633527108 after retry RPC after error@453425601) -> (continuation of Deferred@1601681187 after retry RPC after error@401563549) -> (continuation of Deferred@616869599 after retry RPC after error@178438967) -> (continuation of Deferred@1745789935 after retry RPC after error@99617476) -> (continuation of Deferred@79953651 after retry RPC after error@409222695) -> (continuation of Deferred@90765087 after retry RPC after error@13335388) -> (continuation of Deferred@467937744 after retry RPC after error@575191437) -> (continuation of Deferred@1624924884 after retry RPC after error@999351304) -> (continuation of Deferred@16834563 after retry RPC after error@1726199167) -> (continuation of Deferred@1436334005 after retry RPC after error@1771092394) -> (continuation of Deferred@1872120931 after retry RPC after error@346375331) -> (continuation of Deferred@615807133 after retry RPC after error@1208338504) -> (continuation of Deferred@355299455 after retry RPC after error@731098661) -> (continuation of Deferred@744770394 after retry RPC after error@1254461718) -> (continuation of Deferred@873840313 after retry RPC after error@37870744) -> (continuation of Deferred@217147665 after retry RPC after error@1792672952) -> (continuation of Deferred@1940102956 after retry RPC after error@2139764956) -> (continuation of Deferred@2025091037 after retry RPC after error@1123109815) -> (continuation of Deferred@759490106 after retry RPC after error@52848534) -> (continuation of Deferred@1488403112 after retry RPC after error@282296058)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.189:43805,127.1.19.188:46459,127.1.19.190:45973) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:43805(127.1.19.189:43805): Connection refused: /127.1.19.189:43805,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:45973(127.1.19.190:45973): Connection refused: /127.1.19.190:45973
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.190:45973
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)