06:34:45.015 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
06:34:45.027 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskYC5Dsr/test-tmp
06:34:45.029 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
06:34:45.033 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskYC5Dsr/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
06:34:45.256 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
06:34:45.417 [DEBUG - main] (MiniKuduCluster.java:195) Response:
06:34:45.418 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
06:34:45.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:45.421592 27762 mini_oidc.cc:152] Starting JWKS server
06:34:45.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:45.425581 27762 webserver.cc:460] Webserver started at http://127.0.0.1:40701/ using document root <none> and password file <none>
06:34:45.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:45.426134 27762 mini_oidc.cc:189] Starting OIDC Discovery server
06:34:45.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:45.428503 27762 webserver.cc:460] Webserver started at http://127.0.0.1:46503/ using document root <none> and password file <none>
06:34:45.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T06:34:45Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
06:34:45.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T06:34:45Z Disabled control of system clock
06:34:45.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:45.477344 27762 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:34:45.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:34:45.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-0/wal
06:34:45.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-0/data
06:34:45.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:34:45.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:34:45.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:34:45.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:34:45.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:34:45.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:34:45.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-0/logs
06:34:45.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-0/data/info.pb
06:34:45.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:34:45.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:34:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:34:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:34:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:34:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:34:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:34:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:34:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:34:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:34:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:34:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:34:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.28.190:41449
06:34:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.28.190
06:34:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:34:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.28.148:42517
06:34:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:34:45.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:34:45.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:34:45.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:34:45.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.28.190:41449,127.27.28.189:35445,127.27.28.188:32903
06:34:45.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:34:45.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40701/jwks.json with env {}
06:34:45.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:45.775089 27774 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:34:45.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:45.775629 27774 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:34:45.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:45.776260 27774 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:34:45.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:45.806384 27774 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:34:45.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:45.806658 27774 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:34:45.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:45.806842 27774 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:34:45.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:45.807018 27774 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:34:45.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:45.807304 27774 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:34:45.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:45.807524 27774 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40701/jwks.json
06:34:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:45.841856 27774 master_runner.cc:386] Master server non-default flags:
06:34:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:34:45.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.28.148:42517
06:34:45.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:34:45.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:34:45.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-0/data
06:34:45.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-0/wal
06:34:45.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:34:45.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.28.190:41449,127.27.28.189:35445,127.27.28.188:32903
06:34:45.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:34:45.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:34:45.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:34:45.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.28.190:41449
06:34:45.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:34:45.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:34:45.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:34:45.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40701/jwks.json
06:34:45.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:34:45.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:34:45.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-0/data/info.pb
06:34:45.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.28.190
06:34:45.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:34:45.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:34:45.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:34:45.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:34:45.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:34:45.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:34:45.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-0/logs
06:34:45.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:34:45.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:34:45.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:34:45.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:34:45.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:34:45.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:34:45.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:34:45.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:34:45.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:34:45.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:34:45.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:45.843277 27774 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:34:45.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:45.845364 27774 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:34:45.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:45.865093 27783 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:34:47.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:45.865056 27780 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:34:47.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:45.865058 27781 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:34:47.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:45.865902 27774 server_base.cc:1034] running on GCE node
06:34:47.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.071211 27774 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:34:47.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.074496 27774 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:34:47.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.075914 27774 hybrid_clock.cc:648] HybridClock initialized: now 1790663687075893 us; error 49 us; skew 500 ppm
06:34:47.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.076802 27774 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:34:47.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.091100 27774 webserver.cc:460] Webserver started at http://127.27.28.190:35685/ using document root <none> and password file <none>
06:34:47.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.092005 27774 fs_manager.cc:359] Metadata directory not provided
06:34:47.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.092188 27774 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:34:47.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.092727 27774 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:34:47.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.097958 27774 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-0/data/instance:
06:34:47.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f5dda35177aa4bbba1227f44ca46e9f6"
06:34:47.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:34:47 on dist-test-slave-jqnr"
06:34:47.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.099105 27774 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-0/wal/instance:
06:34:47.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f5dda35177aa4bbba1227f44ca46e9f6"
06:34:47.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:34:47 on dist-test-slave-jqnr"
06:34:47.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.106338 27774 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
06:34:47.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.112311 27790 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:34:47.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.113473 27774 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
06:34:47.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.113898 27774 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-0/data,/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-0/wal
06:34:47.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f5dda35177aa4bbba1227f44ca46e9f6"
06:34:47.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:34:47 on dist-test-slave-jqnr"
06:34:47.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.114377 27774 fs_report.cc:383] FS layout report
06:34:47.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:34:47.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-0/wal
06:34:47.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-0/wal
06:34:47.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-0/data/data
06:34:47.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:34:47.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:34:47.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:34:47.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:34:47.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:34:47.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:34:47.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:34:47.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:34:47.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:34:47.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:34:47.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:34:47.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.160157 27774 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:34:47.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.161907 27774 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:34:47.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.162329 27774 kserver.cc:163] Server-wide thread pool size limit: 3276
06:34:47.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.236999 27774 rpc_server.cc:307] RPC server started. Bound to: 127.27.28.190:41449
06:34:47.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.237074 27841 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.28.190:41449 every 8 connection(s)
06:34:47.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.239856 27774 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-0/data/info.pb
06:34:47.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.241222 27762 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu as pid 27774
06:34:47.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.241724 27762 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-0/wal/instance
06:34:47.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.246244 27762 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:34:47.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:34:47.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-1/wal
06:34:47.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-1/data
06:34:47.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:34:47.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:34:47.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:34:47.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:34:47.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:34:47.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:34:47.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-1/logs
06:34:47.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-1/data/info.pb
06:34:47.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:34:47.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:34:47.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:34:47.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:34:47.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:34:47.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:34:47.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:34:47.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:34:47.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:34:47.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:34:47.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:34:47.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:34:47.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.28.189:35445
06:34:47.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.28.189
06:34:47.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:34:47.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.28.148:42517
06:34:47.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:34:47.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:34:47.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:34:47.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:34:47.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.28.190:41449,127.27.28.189:35445,127.27.28.188:32903
06:34:47.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:34:47.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40701/jwks.json with env {}
06:34:47.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.246622 27842 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:34:47.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.262939 27842 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.28.190" port: 41449 } has no permanent_uuid. Determining permanent_uuid...
06:34:47.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.294974 27842 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } has no permanent_uuid. Determining permanent_uuid...
06:34:47.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:47.297353 27794 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.28.189:35445: connect: Connection refused (error 111)
06:34:47.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:47.301833 27842 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.28.189:35445: Network error: Client connection negotiation failed: client connection to 127.27.28.189:35445: connect: Connection refused (error 111)
06:34:47.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.351397 27842 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } attempt: 1
06:34:47.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:47.355320 27842 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.28.189:35445: Network error: Client connection negotiation failed: client connection to 127.27.28.189:35445: connect: Connection refused (error 111)
06:34:47.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.423781 27842 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } attempt: 2
06:34:47.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:47.427471 27842 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.28.189:35445: Network error: Client connection negotiation failed: client connection to 127.27.28.189:35445: connect: Connection refused (error 111)
06:34:47.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.518996 27842 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } attempt: 3
06:34:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:47.522656 27842 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.28.189:35445: Network error: Client connection negotiation failed: client connection to 127.27.28.189:35445: connect: Connection refused (error 111)
06:34:47.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:47.570261 27844 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:34:47.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:47.570818 27844 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:34:47.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:47.571359 27844 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:34:47.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:47.603356 27844 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:34:47.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:47.603667 27844 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:34:47.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:47.603978 27844 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:34:47.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:47.604285 27844 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:34:47.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:47.604622 27844 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:34:47.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:47.604880 27844 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40701/jwks.json
06:34:47.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.639381 27844 master_runner.cc:386] Master server non-default flags:
06:34:47.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:34:47.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.28.148:42517
06:34:47.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:34:47.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:34:47.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-1/data
06:34:47.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-1/wal
06:34:47.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:34:47.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.28.190:41449,127.27.28.189:35445,127.27.28.188:32903
06:34:47.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:34:47.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:34:47.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:34:47.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.28.189:35445
06:34:47.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:34:47.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:34:47.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:34:47.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40701/jwks.json
06:34:47.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:34:47.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:34:47.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-1/data/info.pb
06:34:47.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.28.189
06:34:47.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:34:47.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:34:47.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:34:47.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:34:47.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:34:47.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:34:47.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-1/logs
06:34:47.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:34:47.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:34:47.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:34:47.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:34:47.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:34:47.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:34:47.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:34:47.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:34:47.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:34:47.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:34:47.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.640620 27844 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:34:47.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.642755 27844 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:34:47.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:47.662889 27856 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:34:47.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.666040 27842 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } attempt: 4
06:34:47.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:47.670411 27842 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.28.189:35445: Network error: Client connection negotiation failed: client connection to 127.27.28.189:35445: connect: Connection refused (error 111)
06:34:47.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.969802 27842 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } attempt: 5
06:34:47.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:47.974373 27842 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.28.189:35445: Network error: Client connection negotiation failed: client connection to 127.27.28.189:35445: connect: Connection refused (error 111)
06:34:48.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:48.522010 27842 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } attempt: 6
06:34:48.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:48.533397 27842 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.28.189:35445: Network error: Client connection negotiation failed: client connection to 127.27.28.189:35445: connect: Connection refused (error 111)
06:34:48.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:47.663605 27853 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:34:48.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:47.664673 27844 server_base.cc:1034] running on GCE node
06:34:48.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:47.664693 27854 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:34:48.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:48.857686 27844 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:34:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:48.860740 27844 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:34:48.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:48.864228 27844 hybrid_clock.cc:648] HybridClock initialized: now 1790663688863562 us; error 682 us; skew 500 ppm
06:34:48.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:48.865345 27844 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:34:48.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:48.872432 27844 webserver.cc:460] Webserver started at http://127.27.28.189:39613/ using document root <none> and password file <none>
06:34:48.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:48.873407 27844 fs_manager.cc:359] Metadata directory not provided
06:34:48.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:48.873588 27844 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:34:48.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:48.873960 27844 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:34:48.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:48.878636 27844 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-1/data/instance:
06:34:48.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3ecda9341f0e4b08a6d1ae520c7438e9"
06:34:48.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:34:48 on dist-test-slave-jqnr"
06:34:48.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:48.879721 27844 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-1/wal/instance:
06:34:48.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3ecda9341f0e4b08a6d1ae520c7438e9"
06:34:48.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:34:48 on dist-test-slave-jqnr"
06:34:48.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:48.886705 27844 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.009s sys 0.000s
06:34:48.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:48.892567 27865 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:34:48.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:48.893669 27844 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
06:34:48.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:48.894026 27844 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-1/data,/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-1/wal
06:34:48.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3ecda9341f0e4b08a6d1ae520c7438e9"
06:34:48.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:34:48 on dist-test-slave-jqnr"
06:34:48.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:48.894405 27844 fs_report.cc:383] FS layout report
06:34:48.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:34:48.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-1/wal
06:34:48.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-1/wal
06:34:48.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-1/data/data
06:34:48.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:34:48.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:34:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:34:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:34:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:34:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:34:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:34:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:34:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:34:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:34:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:34:48.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:48.940114 27844 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:34:48.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:48.941672 27844 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:34:48.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:48.942256 27844 kserver.cc:163] Server-wide thread pool size limit: 3276
06:34:49.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.017588 27844 rpc_server.cc:307] RPC server started. Bound to: 127.27.28.189:35445
06:34:49.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.017678 27916 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.28.189:35445 every 8 connection(s)
06:34:49.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.020357 27844 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-1/data/info.pb
06:34:49.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.024027 27762 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu as pid 27844
06:34:49.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.024509 27762 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-1/wal/instance
06:34:49.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.026679 27917 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:34:49.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.027475 27762 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:34:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:34:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-2/wal
06:34:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-2/data
06:34:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:34:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:34:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:34:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:34:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:34:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:34:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-2/logs
06:34:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-2/data/info.pb
06:34:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:34:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:34:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:34:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:34:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:34:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:34:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:34:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:34:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:34:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:34:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:34:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:34:49.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.28.188:32903
06:34:49.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.28.188
06:34:49.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:34:49.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.28.148:42517
06:34:49.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:34:49.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:34:49.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:34:49.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:34:49.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.28.190:41449,127.27.28.189:35445,127.27.28.188:32903
06:34:49.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:34:49.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40701/jwks.json with env {}
06:34:49.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.041062 27917 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.28.190" port: 41449 } has no permanent_uuid. Determining permanent_uuid...
06:34:49.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.064165 27917 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } has no permanent_uuid. Determining permanent_uuid...
06:34:49.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.081768 27917 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } has no permanent_uuid. Determining permanent_uuid...
06:34:49.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:49.084707 27868 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.28.188:32903: connect: Connection refused (error 111)
06:34:49.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:49.090462 27917 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.28.188:32903: Network error: Client connection negotiation failed: client connection to 127.27.28.188:32903: connect: Connection refused (error 111)
06:34:49.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.140435 27917 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } attempt: 1
06:34:49.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:49.144258 27917 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.28.188:32903: Network error: Client connection negotiation failed: client connection to 127.27.28.188:32903: connect: Connection refused (error 111)
06:34:49.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.212699 27917 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } attempt: 2
06:34:49.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:49.224495 27917 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.28.188:32903: Network error: Client connection negotiation failed: client connection to 127.27.28.188:32903: connect: Connection refused (error 111)
06:34:49.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.316004 27917 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } attempt: 3
06:34:49.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:49.320044 27917 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.28.188:32903: Network error: Client connection negotiation failed: client connection to 127.27.28.188:32903: connect: Connection refused (error 111)
06:34:49.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:49.363408 27919 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:34:49.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:49.364168 27919 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:34:49.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:49.364913 27919 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:34:49.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:49.395576 27919 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:34:49.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:49.395954 27919 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:34:49.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:49.396275 27919 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:34:49.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:49.396642 27919 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:34:49.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:49.396945 27919 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:34:49.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:49.397226 27919 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40701/jwks.json
06:34:49.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.431675 27919 master_runner.cc:386] Master server non-default flags:
06:34:49.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:34:49.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.28.148:42517
06:34:49.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:34:49.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:34:49.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-2/data
06:34:49.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-2/wal
06:34:49.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:34:49.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.28.190:41449,127.27.28.189:35445,127.27.28.188:32903
06:34:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:34:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:34:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:34:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.28.188:32903
06:34:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:34:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:34:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:34:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40701/jwks.json
06:34:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:34:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:34:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-2/data/info.pb
06:34:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.28.188
06:34:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:34:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:34:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:34:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:34:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:34:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:34:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-2/logs
06:34:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:34:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:34:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:34:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:34:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:34:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:34:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:34:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:34:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:34:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:34:49.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.433283 27919 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:34:49.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.435443 27919 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:34:49.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:49.455662 27932 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:34:49.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.463441 27917 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } attempt: 4
06:34:49.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:49.467350 27917 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.28.188:32903: Network error: Client connection negotiation failed: client connection to 127.27.28.188:32903: connect: Connection refused (error 111)
06:34:49.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.594053 27842 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } attempt: 7
06:34:49.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.623148 27842 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } has no permanent_uuid. Determining permanent_uuid...
06:34:49.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:49.629786 27842 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.28.188:32903: Network error: Client connection negotiation failed: client connection to 127.27.28.188:32903: connect: Connection refused (error 111)
06:34:49.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.688422 27842 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } attempt: 1
06:34:49.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:49.693289 27842 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.28.188:32903: Network error: Client connection negotiation failed: client connection to 127.27.28.188:32903: connect: Connection refused (error 111)
06:34:49.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.766727 27917 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } attempt: 5
06:34:49.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:49.771726 27917 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.28.188:32903: Network error: Client connection negotiation failed: client connection to 127.27.28.188:32903: connect: Connection refused (error 111)
06:34:49.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.774693 27842 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } attempt: 2
06:34:49.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:49.778716 27842 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.28.188:32903: Network error: Client connection negotiation failed: client connection to 127.27.28.188:32903: connect: Connection refused (error 111)
06:34:49.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.864114 27842 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } attempt: 3
06:34:49.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:49.868836 27842 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.28.188:32903: Network error: Client connection negotiation failed: client connection to 127.27.28.188:32903: connect: Connection refused (error 111)
06:34:50.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.009289 27842 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } attempt: 4
06:34:50.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:50.013759 27842 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.28.188:32903: Network error: Client connection negotiation failed: client connection to 127.27.28.188:32903: connect: Connection refused (error 111)
06:34:50.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.297191 27842 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } attempt: 5
06:34:50.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:50.301735 27842 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.28.188:32903: Network error: Client connection negotiation failed: client connection to 127.27.28.188:32903: connect: Connection refused (error 111)
06:34:50.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.319149 27917 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } attempt: 6
06:34:50.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:50.327251 27917 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.28.188:32903: Network error: Client connection negotiation failed: client connection to 127.27.28.188:32903: connect: Connection refused (error 111)
06:34:50.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:49.455734 27930 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:34:50.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:49.457080 27919 server_base.cc:1034] running on GCE node
06:34:50.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:49.455727 27929 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:34:50.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.690853 27919 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:34:50.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.693815 27919 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:34:50.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.695174 27919 hybrid_clock.cc:648] HybridClock initialized: now 1790663690695158 us; error 62 us; skew 500 ppm
06:34:50.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.696014 27919 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:34:50.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.702685 27919 webserver.cc:460] Webserver started at http://127.27.28.188:35015/ using document root <none> and password file <none>
06:34:50.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.703589 27919 fs_manager.cc:359] Metadata directory not provided
06:34:50.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.703768 27919 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:34:50.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.704156 27919 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:34:50.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.711177 27919 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-2/data/instance:
06:34:50.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dace4628823648c6ae4f10171cd58522"
06:34:50.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:34:50 on dist-test-slave-jqnr"
06:34:50.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.712770 27919 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-2/wal/instance:
06:34:50.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dace4628823648c6ae4f10171cd58522"
06:34:50.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:34:50 on dist-test-slave-jqnr"
06:34:50.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.722172 27919 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.006s sys 0.003s
06:34:50.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.729460 27944 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:34:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.730573 27919 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
06:34:50.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.730855 27919 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-2/data,/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-2/wal
06:34:50.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dace4628823648c6ae4f10171cd58522"
06:34:50.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:34:50 on dist-test-slave-jqnr"
06:34:50.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.731221 27919 fs_report.cc:383] FS layout report
06:34:50.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:34:50.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-2/wal
06:34:50.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-2/wal
06:34:50.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-2/data/data
06:34:50.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:34:50.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:34:50.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:34:50.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:34:50.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:34:50.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:34:50.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:34:50.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:34:50.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:34:50.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:34:50.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:34:50.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.765861 27919 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:34:50.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.767261 27919 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:34:50.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.767654 27919 kserver.cc:163] Server-wide thread pool size limit: 3276
06:34:50.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.844519 27919 rpc_server.cc:307] RPC server started. Bound to: 127.27.28.188:32903
06:34:50.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.844627 27995 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.28.188:32903 every 8 connection(s)
06:34:50.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.847146 27919 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-2/data/info.pb
06:34:50.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.853493 27996 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:34:50.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.854144 27842 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } attempt: 6
06:34:50.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.857810 27762 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu as pid 27919
06:34:50.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.858249 27762 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/master-2/wal/instance
06:34:50.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.879151 27996 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.28.190" port: 41449 } has no permanent_uuid. Determining permanent_uuid...
06:34:50.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.898681 27842 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6: Bootstrap starting.
06:34:50.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.903656 27996 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } has no permanent_uuid. Determining permanent_uuid...
06:34:50.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.905894 27842 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6: Neither blocks nor log segments found. Creating new log.
06:34:50.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.908885 27842 log.cc:824] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6: Log is configured to *not* fsync() on all Append() calls
06:34:50.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.920073 27842 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6: No bootstrap required, opened a new log
06:34:50.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.920455 27996 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } has no permanent_uuid. Determining permanent_uuid...
06:34:50.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.945423 27996 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dace4628823648c6ae4f10171cd58522: Bootstrap starting.
06:34:50.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.947875 27842 raft_consensus.cc:348] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" member_type: VOTER last_known_addr { host: "127.27.28.190" port: 41449 } } peers { permanent_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9" member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } } peers { permanent_uuid: "dace4628823648c6ae4f10171cd58522" member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } }
06:34:50.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.948499 27842 raft_consensus.cc:374] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:34:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.948693 27842 raft_consensus.cc:729] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f5dda35177aa4bbba1227f44ca46e9f6, State: Initialized, Role: FOLLOWER
06:34:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.949462 27842 consensus_queue.cc:260] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" member_type: VOTER last_known_addr { host: "127.27.28.190" port: 41449 } } peers { permanent_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9" member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } } peers { permanent_uuid: "dace4628823648c6ae4f10171cd58522" member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } }
06:34:50.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.951900 27996 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P dace4628823648c6ae4f10171cd58522: Neither blocks nor log segments found. Creating new log.
06:34:50.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.954473 27996 log.cc:824] T 00000000000000000000000000000000 P dace4628823648c6ae4f10171cd58522: Log is configured to *not* fsync() on all Append() calls
06:34:50.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.955538 27842 sys_catalog.cc:564] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [sys.catalog]: configured and running, proceeding with master startup.
06:34:50.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.959776 28008 sys_catalog.cc:455] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" member_type: VOTER last_known_addr { host: "127.27.28.190" port: 41449 } } peers { permanent_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9" member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } } peers { permanent_uuid: "dace4628823648c6ae4f10171cd58522" member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } } }
06:34:50.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.962277 28008 sys_catalog.cc:458] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [sys.catalog]: This master's current role is: FOLLOWER
06:34:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.963382 28008 raft_consensus.cc:482] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:34:50.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.964777 28008 raft_consensus.cc:504] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" member_type: VOTER last_known_addr { host: "127.27.28.190" port: 41449 } } peers { permanent_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9" member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } } peers { permanent_uuid: "dace4628823648c6ae4f10171cd58522" member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } }
06:34:50.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.972417 28008 leader_election.cc:288] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3ecda9341f0e4b08a6d1ae520c7438e9 (127.27.28.189:35445), dace4628823648c6ae4f10171cd58522 (127.27.28.188:32903)
06:34:50.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:50.973882 27996 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dace4628823648c6ae4f10171cd58522: No bootstrap required, opened a new log
06:34:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.005523 27996 raft_consensus.cc:348] T 00000000000000000000000000000000 P dace4628823648c6ae4f10171cd58522 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" member_type: VOTER last_known_addr { host: "127.27.28.190" port: 41449 } } peers { permanent_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9" member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } } peers { permanent_uuid: "dace4628823648c6ae4f10171cd58522" member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } }
06:34:51.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.006840 27996 raft_consensus.cc:374] T 00000000000000000000000000000000 P dace4628823648c6ae4f10171cd58522 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:34:51.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.007357 27996 raft_consensus.cc:729] T 00000000000000000000000000000000 P dace4628823648c6ae4f10171cd58522 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dace4628823648c6ae4f10171cd58522, State: Initialized, Role: FOLLOWER
06:34:51.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.008288 27996 consensus_queue.cc:260] T 00000000000000000000000000000000 P dace4628823648c6ae4f10171cd58522 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" member_type: VOTER last_known_addr { host: "127.27.28.190" port: 41449 } } peers { permanent_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9" member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } } peers { permanent_uuid: "dace4628823648c6ae4f10171cd58522" member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } }
06:34:51.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.012591 28022 sys_catalog.cc:455] T 00000000000000000000000000000000 P dace4628823648c6ae4f10171cd58522 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" member_type: VOTER last_known_addr { host: "127.27.28.190" port: 41449 } } peers { permanent_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9" member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } } peers { permanent_uuid: "dace4628823648c6ae4f10171cd58522" member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } } }
06:34:51.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.016332 28022 sys_catalog.cc:458] T 00000000000000000000000000000000 P dace4628823648c6ae4f10171cd58522 [sys.catalog]: This master's current role is: FOLLOWER
06:34:51.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.015170 27971 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dace4628823648c6ae4f10171cd58522" is_pre_election: true
06:34:51.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.014410 27996 sys_catalog.cc:564] T 00000000000000000000000000000000 P dace4628823648c6ae4f10171cd58522 [sys.catalog]: configured and running, proceeding with master startup.
06:34:51.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.017289 27971 raft_consensus.cc:2454] T 00000000000000000000000000000000 P dace4628823648c6ae4f10171cd58522 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f5dda35177aa4bbba1227f44ca46e9f6 in term 0.
06:34:51.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.021579 27793 leader_election.cc:302] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: dace4628823648c6ae4f10171cd58522, f5dda35177aa4bbba1227f44ca46e9f6; no voters:
06:34:51.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.023069 28009 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [term 0 FOLLOWER]: Leader pre-election won for term 1
06:34:51.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:51.023113 27950 tablet.cc:2369] T 00000000000000000000000000000000 P dace4628823648c6ae4f10171cd58522: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:34:51.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.023910 28009 raft_consensus.cc:482] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
06:34:51.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.024235 28009 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [term 0 FOLLOWER]: Advancing to term 1
06:34:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.033141 28009 raft_consensus.cc:504] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" member_type: VOTER last_known_addr { host: "127.27.28.190" port: 41449 } } peers { permanent_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9" member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } } peers { permanent_uuid: "dace4628823648c6ae4f10171cd58522" member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } }
06:34:51.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.049041 28009 leader_election.cc:288] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [CANDIDATE]: Term 1 election: Requested vote from peers 3ecda9341f0e4b08a6d1ae520c7438e9 (127.27.28.189:35445), dace4628823648c6ae4f10171cd58522 (127.27.28.188:32903)
06:34:51.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.049695 27971 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dace4628823648c6ae4f10171cd58522"
06:34:51.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.050850 27971 raft_consensus.cc:3037] T 00000000000000000000000000000000 P dace4628823648c6ae4f10171cd58522 [term 0 FOLLOWER]: Advancing to term 1
06:34:51.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:51.057809 28021 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:34:51.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.065927 27971 raft_consensus.cc:2454] T 00000000000000000000000000000000 P dace4628823648c6ae4f10171cd58522 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f5dda35177aa4bbba1227f44ca46e9f6 in term 1.
06:34:51.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:51.066699 28021 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:34:51.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.016384 27774 master_runner.cc:186] Error getting master registration for 127.27.28.189:35445: OK, instance_id { permanent_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9" instance_seqno: 1790663688995865 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:34:51.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.070667 27774 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:34:51.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.074824 27793 leader_election.cc:302] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: dace4628823648c6ae4f10171cd58522, f5dda35177aa4bbba1227f44ca46e9f6; no voters:
06:34:51.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.075897 28009 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [term 1 FOLLOWER]: Leader election won for term 1
06:34:51.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.076438 28009 raft_consensus.cc:686] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [term 1 LEADER]: Becoming Leader. State: Replica: f5dda35177aa4bbba1227f44ca46e9f6, State: Running, Role: LEADER
06:34:51.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.077421 28009 consensus_queue.cc:237] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" member_type: VOTER last_known_addr { host: "127.27.28.190" port: 41449 } } peers { permanent_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9" member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } } peers { permanent_uuid: "dace4628823648c6ae4f10171cd58522" member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } }
06:34:51.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.098285 28008 sys_catalog.cc:455] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f5dda35177aa4bbba1227f44ca46e9f6. Latest consensus state: current_term: 1 leader_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" member_type: VOTER last_known_addr { host: "127.27.28.190" port: 41449 } } peers { permanent_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9" member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } } peers { permanent_uuid: "dace4628823648c6ae4f10171cd58522" member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } } }
06:34:51.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.099212 28008 sys_catalog.cc:458] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [sys.catalog]: This master's current role is: LEADER
06:34:51.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.121589 28033 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
06:34:51.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.123606 28036 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
06:34:51.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.131135 28033 catalog_manager.cc:1484] Initializing Kudu cluster ID...
06:34:51.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.157696 27971 raft_consensus.cc:1261] T 00000000000000000000000000000000 P dace4628823648c6ae4f10171cd58522 [term 1 FOLLOWER]: Refusing update from remote peer f5dda35177aa4bbba1227f44ca46e9f6: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:34:51.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.159219 28009 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dace4628823648c6ae4f10171cd58522" member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:34:51.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:51.171663 27796 tablet.cc:2369] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:34:51.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.137070 27919 master_runner.cc:186] Error getting master registration for 127.27.28.189:35445: OK, instance_id { permanent_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9" instance_seqno: 1790663688995865 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:34:51.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.183588 27919 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:34:51.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.204762 28022 sys_catalog.cc:455] T 00000000000000000000000000000000 P dace4628823648c6ae4f10171cd58522 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f5dda35177aa4bbba1227f44ca46e9f6. Latest consensus state: current_term: 1 leader_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" member_type: VOTER last_known_addr { host: "127.27.28.190" port: 41449 } } peers { permanent_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9" member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } } peers { permanent_uuid: "dace4628823648c6ae4f10171cd58522" member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } } }
06:34:51.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.205325 28022 sys_catalog.cc:458] T 00000000000000000000000000000000 P dace4628823648c6ae4f10171cd58522 [sys.catalog]: This master's current role is: FOLLOWER
06:34:51.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.202728 28008 sys_catalog.cc:455] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" member_type: VOTER last_known_addr { host: "127.27.28.190" port: 41449 } } peers { permanent_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9" member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } } peers { permanent_uuid: "dace4628823648c6ae4f10171cd58522" member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } } }
06:34:51.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.206743 28008 sys_catalog.cc:458] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [sys.catalog]: This master's current role is: LEADER
06:34:51.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.207346 28037 mvcc.cc:205] Tried to move back new op lower bound from 7334558478958202880 to 7334558478663385088. Current Snapshot: MvccSnapshot[applied={T|T < 7334558478958202880}]
06:34:51.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.210443 28033 catalog_manager.cc:1347] Generated new cluster ID: 698e891f31c8437c951f2c10ff2a83a4
06:34:51.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.210709 28033 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
06:34:51.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.213219 28022 sys_catalog.cc:455] T 00000000000000000000000000000000 P dace4628823648c6ae4f10171cd58522 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" member_type: VOTER last_known_addr { host: "127.27.28.190" port: 41449 } } peers { permanent_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9" member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } } peers { permanent_uuid: "dace4628823648c6ae4f10171cd58522" member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } } }
06:34:51.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.213937 28022 sys_catalog.cc:458] T 00000000000000000000000000000000 P dace4628823648c6ae4f10171cd58522 [sys.catalog]: This master's current role is: FOLLOWER
06:34:51.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.256024 28033 catalog_manager.cc:1370] Generated new certificate authority record
06:34:51.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.259022 28033 catalog_manager.cc:1504] Loading token signing keys...
06:34:51.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.292693 28033 catalog_manager.cc:5868] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6: Generated new TSK 0
06:34:51.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.294886 28033 catalog_manager.cc:1514] Initializing in-progress tserver states...
06:34:51.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.312817 27762 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:34:51.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:34:51.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-0/wal
06:34:51.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-0/data
06:34:51.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:34:51.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:34:51.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:34:51.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:34:51.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:34:51.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:34:51.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-0/logs
06:34:51.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-0/data/info.pb
06:34:51.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:34:51.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:34:51.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:34:51.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:34:51.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:34:51.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:34:51.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:34:51.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:34:51.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:34:51.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:34:51.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.28.129:0
06:34:51.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.28.129
06:34:51.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.28.129
06:34:51.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:34:51.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.28.190:41449,127.27.28.189:35445,127.27.28.188:32903
06:34:51.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.28.148:42517
06:34:51.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:34:51.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:34:51.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:34:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.387658 27917 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } attempt: 7
06:34:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.419909 27917 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3ecda9341f0e4b08a6d1ae520c7438e9: Bootstrap starting.
06:34:51.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.427397 27917 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3ecda9341f0e4b08a6d1ae520c7438e9: Neither blocks nor log segments found. Creating new log.
06:34:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.430680 27917 log.cc:824] T 00000000000000000000000000000000 P 3ecda9341f0e4b08a6d1ae520c7438e9: Log is configured to *not* fsync() on all Append() calls
06:34:51.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.441520 27917 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3ecda9341f0e4b08a6d1ae520c7438e9: No bootstrap required, opened a new log
06:34:51.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.475066 27917 raft_consensus.cc:348] T 00000000000000000000000000000000 P 3ecda9341f0e4b08a6d1ae520c7438e9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" member_type: VOTER last_known_addr { host: "127.27.28.190" port: 41449 } } peers { permanent_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9" member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } } peers { permanent_uuid: "dace4628823648c6ae4f10171cd58522" member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } }
06:34:51.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.476065 27917 raft_consensus.cc:374] T 00000000000000000000000000000000 P 3ecda9341f0e4b08a6d1ae520c7438e9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:34:51.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.476331 27917 raft_consensus.cc:729] T 00000000000000000000000000000000 P 3ecda9341f0e4b08a6d1ae520c7438e9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3ecda9341f0e4b08a6d1ae520c7438e9, State: Initialized, Role: FOLLOWER
06:34:51.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.477437 27917 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3ecda9341f0e4b08a6d1ae520c7438e9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" member_type: VOTER last_known_addr { host: "127.27.28.190" port: 41449 } } peers { permanent_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9" member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } } peers { permanent_uuid: "dace4628823648c6ae4f10171cd58522" member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } }
06:34:51.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.483369 27917 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3ecda9341f0e4b08a6d1ae520c7438e9 [sys.catalog]: configured and running, proceeding with master startup.
06:34:51.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.483539 27891 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9"
06:34:51.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.484699 27891 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 3ecda9341f0e4b08a6d1ae520c7438e9 [term 0 FOLLOWER]: Advancing to term 1
06:34:51.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.488926 27892 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9" is_pre_election: true
06:34:51.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.499886 27891 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 3ecda9341f0e4b08a6d1ae520c7438e9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f5dda35177aa4bbba1227f44ca46e9f6 in term 1.
06:34:51.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.502107 27890 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 3ecda9341f0e4b08a6d1ae520c7438e9 [term 1 FOLLOWER]: Refusing update from remote peer f5dda35177aa4bbba1227f44ca46e9f6: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:34:51.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.482163 28046 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3ecda9341f0e4b08a6d1ae520c7438e9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" member_type: VOTER last_known_addr { host: "127.27.28.190" port: 41449 } } peers { permanent_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9" member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } } peers { permanent_uuid: "dace4628823648c6ae4f10171cd58522" member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } } }
06:34:51.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.512564 28046 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3ecda9341f0e4b08a6d1ae520c7438e9 [sys.catalog]: This master's current role is: FOLLOWER
06:34:51.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.516688 28009 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9" member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
06:34:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.603178 28054 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3ecda9341f0e4b08a6d1ae520c7438e9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" member_type: VOTER last_known_addr { host: "127.27.28.190" port: 41449 } } peers { permanent_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9" member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } } peers { permanent_uuid: "dace4628823648c6ae4f10171cd58522" member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } } }
06:34:51.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.605420 28054 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3ecda9341f0e4b08a6d1ae520c7438e9 [sys.catalog]: This master's current role is: FOLLOWER
06:34:51.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.604691 28046 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3ecda9341f0e4b08a6d1ae520c7438e9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f5dda35177aa4bbba1227f44ca46e9f6. Latest consensus state: current_term: 1 leader_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" member_type: VOTER last_known_addr { host: "127.27.28.190" port: 41449 } } peers { permanent_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9" member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } } peers { permanent_uuid: "dace4628823648c6ae4f10171cd58522" member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } } }
06:34:51.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.606132 28046 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3ecda9341f0e4b08a6d1ae520c7438e9 [sys.catalog]: This master's current role is: FOLLOWER
06:34:51.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.642122 28062 catalog_manager.cc:1259] Loaded cluster ID: 698e891f31c8437c951f2c10ff2a83a4
06:34:51.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.643417 28062 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 3ecda9341f0e4b08a6d1ae520c7438e9: loading cluster ID for follower catalog manager: success
06:34:51.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.651710 28062 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 3ecda9341f0e4b08a6d1ae520c7438e9: acquiring CA information for follower catalog manager: success
06:34:51.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.658433 28062 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 3ecda9341f0e4b08a6d1ae520c7438e9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:34:51.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:51.817503 28043 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:34:51.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:51.818169 28043 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:34:51.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:51.818951 28043 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:34:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:51.876216 28043 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:34:51.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:51.877614 28043 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.28.129
06:34:51.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.940182 28043 tablet_server_runner.cc:78] Tablet server non-default flags:
06:34:51.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:34:51.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.28.148:42517
06:34:51.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:34:51.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:34:51.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-0/data
06:34:51.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-0/wal
06:34:51.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:34:51.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:34:51.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.28.129:0
06:34:51.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:34:51.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:34:51.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:34:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-0/data/info.pb
06:34:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.28.129
06:34:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:34:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.28.190:41449,127.27.28.189:35445,127.27.28.188:32903
06:34:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:34:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:34:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:34:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:34:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:34:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.28.129
06:34:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-0/logs
06:34:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:34:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:34:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:34:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:34:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:34:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:34:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:34:51.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:34:51.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:34:51.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:34:51.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.941754 28043 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:34:51.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:51.943819 28043 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:34:51.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:51.970108 28070 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:34:52.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:52.007309 28009 sys_catalog.cc:455] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5dda35177aa4bbba1227f44ca46e9f6" member_type: VOTER last_known_addr { host: "127.27.28.190" port: 41449 } } peers { permanent_uuid: "3ecda9341f0e4b08a6d1ae520c7438e9" member_type: VOTER last_known_addr { host: "127.27.28.189" port: 35445 } } peers { permanent_uuid: "dace4628823648c6ae4f10171cd58522" member_type: VOTER last_known_addr { host: "127.27.28.188" port: 32903 } } }
06:34:52.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:52.008111 28009 sys_catalog.cc:458] T 00000000000000000000000000000000 P f5dda35177aa4bbba1227f44ca46e9f6 [sys.catalog]: This master's current role is: LEADER
06:34:52.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:52.145188 28036 catalog_manager.cc:1259] Loaded cluster ID: 698e891f31c8437c951f2c10ff2a83a4
06:34:52.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:52.145471 28036 catalog_manager.cc:1552] T 00000000000000000000000000000000 P dace4628823648c6ae4f10171cd58522: loading cluster ID for follower catalog manager: success
06:34:52.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:52.163033 28036 catalog_manager.cc:1574] T 00000000000000000000000000000000 P dace4628823648c6ae4f10171cd58522: acquiring CA information for follower catalog manager: success
06:34:52.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:52.168699 28036 catalog_manager.cc:1602] T 00000000000000000000000000000000 P dace4628823648c6ae4f10171cd58522: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:34:53.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:51.981021 28072 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:34:53.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:53.489725 28071 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
06:34:53.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.489836 28043 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:34:53.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:51.979014 28069 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:34:53.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.494625 28043 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:34:53.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.506737 28043 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:34:53.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.508289 28043 hybrid_clock.cc:648] HybridClock initialized: now 1790663693508158 us; error 64 us; skew 500 ppm
06:34:53.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.509397 28043 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:34:53.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.545224 28043 webserver.cc:460] Webserver started at http://127.27.28.129:33237/ using document root <none> and password file <none>
06:34:53.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.546494 28043 fs_manager.cc:359] Metadata directory not provided
06:34:53.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.546763 28043 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:34:53.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.547302 28043 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:34:53.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.554605 28043 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-0/data/instance:
06:34:53.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7dc4727adf2a451797e89cd3e988e7be"
06:34:53.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:34:53 on dist-test-slave-jqnr"
06:34:53.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.556072 28043 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-0/wal/instance:
06:34:53.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7dc4727adf2a451797e89cd3e988e7be"
06:34:53.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:34:53 on dist-test-slave-jqnr"
06:34:53.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.571159 28043 fs_manager.cc:691] Time spent creating directory manager: real 0.014s user 0.008s sys 0.004s
06:34:53.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.584838 28083 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:34:53.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.586418 28043 fs_manager.cc:725] Time spent opening block manager: real 0.011s user 0.011s sys 0.000s
06:34:53.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.586804 28043 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-0/data,/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-0/wal
06:34:53.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7dc4727adf2a451797e89cd3e988e7be"
06:34:53.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:34:53 on dist-test-slave-jqnr"
06:34:53.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.587224 28043 fs_report.cc:383] FS layout report
06:34:53.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:34:53.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-0/wal
06:34:53.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-0/wal
06:34:53.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-0/data/data
06:34:53.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:34:53.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:34:53.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:34:53.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:34:53.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:34:53.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:34:53.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:34:53.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:34:53.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:34:53.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:34:53.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:34:53.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.676259 28043 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:34:53.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.678304 28043 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:34:53.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.678938 28043 kserver.cc:163] Server-wide thread pool size limit: 3276
06:34:53.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.697152 28043 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:34:53.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.709105 28043 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:34:53.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.709417 28043 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
06:34:53.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.709705 28043 ts_tablet_manager.cc:610] Registered 0 tablets
06:34:53.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:53.709903 28043 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
06:34:54.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:54.150082 27992 debug-util.cc:398] Leaking SignalData structure 0x7b08000a1b20 after lost signal to thread 27924
06:34:54.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:54.150725 27992 debug-util.cc:398] Leaking SignalData structure 0x7b08000b93a0 after lost signal to thread 27995
06:34:54.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:54.380561 28043 rpc_server.cc:307] RPC server started. Bound to: 127.27.28.129:38651
06:34:54.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:54.384465 28043 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-0/data/info.pb
06:34:54.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:54.384922 28195 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.28.129:38651 every 8 connection(s)
06:34:54.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:54.392969 27762 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu as pid 28043
06:34:54.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:54.393458 27762 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-0/wal/instance
06:34:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:54.406816 27762 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:34:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:34:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-1/wal
06:34:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-1/data
06:34:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:34:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:34:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:34:54.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:34:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:34:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:34:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-1/logs
06:34:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-1/data/info.pb
06:34:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:34:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:34:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:34:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:34:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:34:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:34:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:34:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:34:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:34:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:34:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.28.130:0
06:34:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.28.130
06:34:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.28.130
06:34:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:34:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.28.190:41449,127.27.28.189:35445,127.27.28.188:32903
06:34:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.28.148:42517
06:34:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:34:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:34:54.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:34:54.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:54.524168 28196 heartbeater.cc:347] Connected to a master server at 127.27.28.189:35445
06:34:54.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:54.524708 28196 heartbeater.cc:464] Registering TS with master...
06:34:54.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:54.526306 28196 heartbeater.cc:511] Master 127.27.28.189:35445 requested a full tablet report, sending...
06:34:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:54.529798 27882 ts_manager.cc:194] Registered new tserver with Master: 7dc4727adf2a451797e89cd3e988e7be (127.27.28.129:38651)
06:34:54.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:54.531060 28203 heartbeater.cc:347] Connected to a master server at 127.27.28.190:41449
06:34:54.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:54.531347 28203 heartbeater.cc:464] Registering TS with master...
06:34:54.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:54.532152 28203 heartbeater.cc:511] Master 127.27.28.190:41449 requested a full tablet report, sending...
06:34:54.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:54.534575 27807 ts_manager.cc:194] Registered new tserver with Master: 7dc4727adf2a451797e89cd3e988e7be (127.27.28.129:38651)
06:34:54.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:54.537442 27807 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.28.129:43807
06:34:54.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:54.545606 28199 heartbeater.cc:347] Connected to a master server at 127.27.28.188:32903
06:34:54.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:54.545912 28199 heartbeater.cc:464] Registering TS with master...
06:34:54.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:54.546579 28199 heartbeater.cc:511] Master 127.27.28.188:32903 requested a full tablet report, sending...
06:34:54.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:54.549978 27961 ts_manager.cc:194] Registered new tserver with Master: 7dc4727adf2a451797e89cd3e988e7be (127.27.28.129:38651)
06:34:54.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:54.988394 28202 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:34:54.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:54.989377 28202 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:34:54.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:54.990368 28202 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:34:55.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:55.048851 28202 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:34:55.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:55.050845 28202 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.28.130
06:34:55.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:55.113250 28202 tablet_server_runner.cc:78] Tablet server non-default flags:
06:34:55.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:34:55.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.28.148:42517
06:34:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:34:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:34:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-1/data
06:34:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-1/wal
06:34:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:34:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:34:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.28.130:0
06:34:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:34:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:34:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:34:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-1/data/info.pb
06:34:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.28.130
06:34:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:34:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.28.190:41449,127.27.28.189:35445,127.27.28.188:32903
06:34:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:34:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:34:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:34:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:34:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:34:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.28.130
06:34:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-1/logs
06:34:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:34:55.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:34:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:34:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:34:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:34:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:34:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:34:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:34:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:34:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:34:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:55.117468 28202 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:34:55.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:55.119638 28202 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:34:55.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:55.150573 28213 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:34:55.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:55.542695 28203 heartbeater.cc:503] Master 127.27.28.190:41449 was elected leader, sending a full tablet report...
06:34:56.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:55.151115 28212 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:34:56.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:55.162473 28215 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:34:56.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:56.447794 28214 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
06:34:56.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.447952 28202 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:34:56.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.452227 28202 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:34:56.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.455053 28202 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:34:56.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.456491 28202 hybrid_clock.cc:648] HybridClock initialized: now 1790663696456378 us; error 108 us; skew 500 ppm
06:34:56.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.457679 28202 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:34:56.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.466073 28202 webserver.cc:460] Webserver started at http://127.27.28.130:37085/ using document root <none> and password file <none>
06:34:56.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.467473 28202 fs_manager.cc:359] Metadata directory not provided
06:34:56.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.467821 28202 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:34:56.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.468400 28202 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:34:56.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.476112 28202 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-1/data/instance:
06:34:56.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e82625755cd2490bb4c4915494662f84"
06:34:56.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:34:56 on dist-test-slave-jqnr"
06:34:56.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.477823 28202 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-1/wal/instance:
06:34:56.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e82625755cd2490bb4c4915494662f84"
06:34:56.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:34:56 on dist-test-slave-jqnr"
06:34:56.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.487957 28202 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.004s sys 0.004s
06:34:56.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.495500 28224 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:34:56.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.496803 28202 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
06:34:56.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.497224 28202 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-1/data,/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-1/wal
06:34:56.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e82625755cd2490bb4c4915494662f84"
06:34:56.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:34:56 on dist-test-slave-jqnr"
06:34:56.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.497680 28202 fs_report.cc:383] FS layout report
06:34:56.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:34:56.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-1/wal
06:34:56.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-1/wal
06:34:56.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-1/data/data
06:34:56.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:34:56.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:34:56.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:34:56.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:34:56.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:34:56.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:34:56.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:34:56.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:34:56.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:34:56.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:34:56.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:34:56.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.559674 28202 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:34:56.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.561512 28202 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:34:56.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.562063 28202 kserver.cc:163] Server-wide thread pool size limit: 3276
06:34:56.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.565186 28202 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:34:56.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.570510 28202 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:34:56.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.570796 28202 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
06:34:56.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.571062 28202 ts_tablet_manager.cc:610] Registered 0 tablets
06:34:56.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.571264 28202 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
06:34:56.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.721504 28202 rpc_server.cc:307] RPC server started. Bound to: 127.27.28.130:38033
06:34:56.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.721594 28336 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.28.130:38033 every 8 connection(s)
06:34:56.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.724323 28202 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-1/data/info.pb
06:34:56.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.734719 27762 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu as pid 28202
06:34:56.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.735492 27762 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-1/wal/instance
06:34:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.754766 27762 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:34:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:34:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-2/wal
06:34:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-2/data
06:34:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:34:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:34:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:34:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:34:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:34:56.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:34:56.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-2/logs
06:34:56.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-2/data/info.pb
06:34:56.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:34:56.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:34:56.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:34:56.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:34:56.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:34:56.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:34:56.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:34:56.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:34:56.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:34:56.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:34:56.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.28.131:0
06:34:56.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.28.131
06:34:56.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.28.131
06:34:56.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:34:56.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.28.190:41449,127.27.28.189:35445,127.27.28.188:32903
06:34:56.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.28.148:42517
06:34:56.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:34:56.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:34:56.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
06:34:56.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.775852 28337 heartbeater.cc:347] Connected to a master server at 127.27.28.189:35445
06:34:56.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.776310 28337 heartbeater.cc:464] Registering TS with master...
06:34:56.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.777500 28337 heartbeater.cc:511] Master 127.27.28.189:35445 requested a full tablet report, sending...
06:34:56.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.780548 27882 ts_manager.cc:194] Registered new tserver with Master: e82625755cd2490bb4c4915494662f84 (127.27.28.130:38033)
06:34:56.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.780682 28339 heartbeater.cc:347] Connected to a master server at 127.27.28.190:41449
06:34:56.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.780978 28339 heartbeater.cc:464] Registering TS with master...
06:34:56.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.781749 28339 heartbeater.cc:511] Master 127.27.28.190:41449 requested a full tablet report, sending...
06:34:56.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.783841 28338 heartbeater.cc:347] Connected to a master server at 127.27.28.188:32903
06:34:56.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.784186 28338 heartbeater.cc:464] Registering TS with master...
06:34:56.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.784781 27807 ts_manager.cc:194] Registered new tserver with Master: e82625755cd2490bb4c4915494662f84 (127.27.28.130:38033)
06:34:56.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.785015 28338 heartbeater.cc:511] Master 127.27.28.188:32903 requested a full tablet report, sending...
06:34:56.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.787292 27807 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.28.130:57169
06:34:56.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:56.790076 27961 ts_manager.cc:194] Registered new tserver with Master: e82625755cd2490bb4c4915494662f84 (127.27.28.130:38033)
06:34:57.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:57.075042 28347 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:34:57.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:57.075500 28347 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:34:57.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:57.075904 28347 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:34:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:57.106580 28347 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:34:57.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:57.107355 28347 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.28.131
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:57.142653 28347 tablet_server_runner.cc:78] Tablet server non-default flags:
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.28.148:42517
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-2/data
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-2/wal
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.28.131:0
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-2/data/info.pb
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.28.131
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.28.190:41449,127.27.28.189:35445,127.27.28.188:32903
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.28.131
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-2/logs
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:34:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:34:57.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:57.143788 28347 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:34:57.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:57.145385 28347 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:34:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:57.165205 28356 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:34:57.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:57.791828 28339 heartbeater.cc:503] Master 127.27.28.190:41449 was elected leader, sending a full tablet report...
06:34:58.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:57.165215 28353 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:34:58.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:34:57.166278 28354 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:34:58.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:57.167042 28347 server_base.cc:1034] running on GCE node
06:34:58.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.401723 28347 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:34:58.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.404498 28347 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:34:58.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.405980 28347 hybrid_clock.cc:648] HybridClock initialized: now 1790663698405934 us; error 47 us; skew 500 ppm
06:34:58.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.406826 28347 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:34:58.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.414000 28347 webserver.cc:460] Webserver started at http://127.27.28.131:39597/ using document root <none> and password file <none>
06:34:58.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.414965 28347 fs_manager.cc:359] Metadata directory not provided
06:34:58.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.415160 28347 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:34:58.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.415561 28347 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:34:58.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.420708 28347 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-2/data/instance:
06:34:58.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8c2e0f1efab6434ea85dd219e1a2046d"
06:34:58.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:34:58 on dist-test-slave-jqnr"
06:34:58.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.421869 28347 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-2/wal/instance:
06:34:58.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8c2e0f1efab6434ea85dd219e1a2046d"
06:34:58.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:34:58 on dist-test-slave-jqnr"
06:34:58.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.429584 28347 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
06:34:58.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.435292 28364 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:34:58.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.436488 28347 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
06:34:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.436784 28347 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-2/data,/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-2/wal
06:34:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8c2e0f1efab6434ea85dd219e1a2046d"
06:34:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:34:58 on dist-test-slave-jqnr"
06:34:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.437093 28347 fs_report.cc:383] FS layout report
06:34:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:34:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-2/wal
06:34:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-2/wal
06:34:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-2/data/data
06:34:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:34:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:34:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:34:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:34:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:34:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:34:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:34:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:34:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:34:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:34:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:34:58.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.484144 28347 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:34:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.485746 28347 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:34:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.486320 28347 kserver.cc:163] Server-wide thread pool size limit: 3276
06:34:58.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.488772 28347 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
06:34:58.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.492817 28347 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:34:58.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.493009 28347 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
06:34:58.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.493197 28347 ts_tablet_manager.cc:610] Registered 0 tablets
06:34:58.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.493332 28347 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
06:34:58.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.649169 28347 rpc_server.cc:307] RPC server started. Bound to: 127.27.28.131:39199
06:34:58.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.649386 28476 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.28.131:39199 every 8 connection(s)
06:34:58.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.651752 28347 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-2/data/info.pb
06:34:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.653997 27762 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu as pid 28347
06:34:58.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.654472 27762 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster5222270598785402839/ts-2/wal/instance
06:34:58.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.710909 28477 heartbeater.cc:347] Connected to a master server at 127.27.28.189:35445
06:34:58.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.711354 28477 heartbeater.cc:464] Registering TS with master...
06:34:58.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.712975 28477 heartbeater.cc:511] Master 127.27.28.189:35445 requested a full tablet report, sending...
06:34:58.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.716173 27882 ts_manager.cc:194] Registered new tserver with Master: 8c2e0f1efab6434ea85dd219e1a2046d (127.27.28.131:39199)
06:34:58.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.716630 28478 heartbeater.cc:347] Connected to a master server at 127.27.28.188:32903
06:34:58.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.717429 28478 heartbeater.cc:464] Registering TS with master...
06:34:58.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.718171 28478 heartbeater.cc:511] Master 127.27.28.188:32903 requested a full tablet report, sending...
06:34:58.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.721220 27961 ts_manager.cc:194] Registered new tserver with Master: 8c2e0f1efab6434ea85dd219e1a2046d (127.27.28.131:39199)
06:34:58.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.723896 28479 heartbeater.cc:347] Connected to a master server at 127.27.28.190:41449
06:34:58.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.724225 28479 heartbeater.cc:464] Registering TS with master...
06:34:58.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.724864 28479 heartbeater.cc:511] Master 127.27.28.190:41449 requested a full tablet report, sending...
06:34:58.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.727165 27807 ts_manager.cc:194] Registered new tserver with Master: 8c2e0f1efab6434ea85dd219e1a2046d (127.27.28.131:39199)
06:34:58.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.730491 27807 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.28.131:60227
06:34:58.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:58.739056 27762 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
06:34:58.749 [DEBUG - main] (MiniKuduCluster.java:195) Response:
06:34:58.751 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
06:34:58.762 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.27.28.190"
port: 41449
}
bound_http_address {
host: "127.27.28.190"
port: 35685
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.27.28.189"
port: 35445
}
bound_http_address {
host: "127.27.28.189"
port: 39613
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.27.28.188"
port: 32903
}
bound_http_address {
host: "127.27.28.188"
port: 35015
}
}
}
06:34:58.767 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
06:34:58.769 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.27.28.129"
port: 38651
}
bound_http_address {
host: "127.27.28.129"
port: 33237
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.27.28.130"
port: 38033
}
bound_http_address {
host: "127.27.28.130"
port: 37085
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.27.28.131"
port: 39199
}
bound_http_address {
host: "127.27.28.131"
port: 39597
}
}
}
06:34:58.769 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
06:34:58.937 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
06:34:59.059 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskYC5Dsr/test-tmp
06:34:59.060 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
06:34:59.060 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskYC5Dsr/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
06:34:59.063 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
06:34:59.458 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:34:59.459 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
06:34:59.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:59.461875 28488 mini_oidc.cc:152] Starting JWKS server
06:34:59.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:59.466152 28488 webserver.cc:460] Webserver started at http://127.0.0.1:39537/ using document root <none> and password file <none>
06:34:59.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:59.466734 28488 mini_oidc.cc:189] Starting OIDC Discovery server
06:34:59.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:59.468441 28488 webserver.cc:460] Webserver started at http://127.0.0.1:40617/ using document root <none> and password file <none>
06:34:59.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:34:59.734813 28479 heartbeater.cc:503] Master 127.27.28.190:41449 was elected leader, sending a full tablet report...
06:35:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
06:35:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/principal' for realm 'KRBTEST.COM',
06:35:00.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
06:35:00.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:00 dist-test-slave-jqnr krb5kdc[28501](info): setting up network...
06:35:00.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
06:35:00.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:00 dist-test-slave-jqnr krb5kdc[28501](info): set up 2 sockets
06:35:00.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:00 dist-test-slave-jqnr krb5kdc[28501](info): commencing operation
06:35:00.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
06:35:01.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:01.254498 28488 mini_kdc.cc:121] Time spent starting KDC: real 1.760s user 0.003s sys 0.016s
06:35:01.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
06:35:01.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
06:35:01.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
06:35:01.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
06:35:01.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
06:35:01.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
06:35:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
06:35:01.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
06:35:01.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
06:35:01.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
06:35:01.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/test-user.keytab.
06:35:01.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/test-user.keytab.
06:35:01.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:01 dist-test-slave-jqnr krb5kdc[28501](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790663701, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:35:01.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
06:35:01.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T06:35:01Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
06:35:01.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T06:35:01Z Disabled control of system clock
06:35:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.27.210.62@KRBTEST.COM; defaulting to no policy
06:35:01.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:35:01.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.27.210.62@KRBTEST.COM" created.
06:35:01.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:35:01.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.210.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kudu_127.27.210.62.keytab.
06:35:01.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.210.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kudu_127.27.210.62.keytab.
06:35:01.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:01.449883 28488 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:35:01.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:35:01.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/wal
06:35:01.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/data
06:35:01.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:35:01.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:35:01.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:35:01.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:35:01.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:35:01.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:35:01.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/logs
06:35:01.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/data/info.pb
06:35:01.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:35:01.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:35:01.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:35:01.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:35:01.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:35:01.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:35:01.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:35:01.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:35:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:35:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:35:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:35:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:35:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.210.62:34959
06:35:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.210.62
06:35:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:35:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kudu_127.27.210.62.keytab
06:35:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.210.62
06:35:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:35:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:35:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:35:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.210.20:43351
06:35:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:35:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:35:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:35:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:35:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.210.62:34959,127.27.210.61:44953,127.27.210.60:45231
06:35:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:35:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39537/jwks.json
06:35:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:35:01.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:01.764230 28514 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:35:01.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:01.765065 28514 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:35:01.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:01.765923 28514 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:35:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:01.799614 28514 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:35:01.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:01.799930 28514 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:35:01.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:01.800360 28514 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:35:01.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:01.800693 28514 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:35:01.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:01.801010 28514 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:35:01.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:01.801373 28514 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:35:01.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:01.801602 28514 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39537/jwks.json
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:01.839597 28514 master_runner.cc:386] Master server non-default flags:
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.210.20:43351
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/data
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/wal
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.210.62:34959,127.27.210.61:44953,127.27.210.60:45231
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kudu_127.27.210.62.keytab
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.210.62
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.210.62:34959
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39537/jwks.json
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:35:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/data/info.pb
06:35:01.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.210.62
06:35:01.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:35:01.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:35:01.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:35:01.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:35:01.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:35:01.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:35:01.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/logs
06:35:01.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:35:01.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:35:01.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:35:01.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:35:01.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:35:01.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:35:01.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:35:01.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:35:01.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:35:01.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:35:01.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:01.841102 28514 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:35:01.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:01.842283 28514 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:35:01.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:01.861347 28525 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:35:03.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:01.861869 28523 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:35:03.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:01.861438 28522 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:35:03.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:03.124409 28524 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
06:35:03.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.124451 28514 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:35:03.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.128544 28514 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:35:03.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.131659 28514 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:35:03.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.133098 28514 hybrid_clock.cc:648] HybridClock initialized: now 1790663703133066 us; error 66 us; skew 500 ppm
06:35:03.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.133881 28514 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:35:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:03 dist-test-slave-jqnr krb5kdc[28501](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790663703, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.210.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:35:03.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.139851 28514 init.cc:377] Logged in from keytab as kudu/127.27.210.62@KRBTEST.COM (short username kudu)
06:35:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.145431 28514 webserver.cc:460] Webserver started at http://127.27.210.62:45111/ using document root <none> and password file <none>
06:35:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.146276 28514 fs_manager.cc:359] Metadata directory not provided
06:35:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.146453 28514 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:35:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.146818 28514 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:35:03.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.151705 28514 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/data/instance:
06:35:03.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "912cbf41d7554146b0bd563d78e5ad17"
06:35:03.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:35:03 on dist-test-slave-jqnr"
06:35:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.152871 28514 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/wal/instance:
06:35:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "912cbf41d7554146b0bd563d78e5ad17"
06:35:03.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:35:03 on dist-test-slave-jqnr"
06:35:03.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.160470 28514 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.002s sys 0.005s
06:35:03.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.166410 28533 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:35:03.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.167599 28514 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
06:35:03.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.167929 28514 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/data,/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/wal
06:35:03.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "912cbf41d7554146b0bd563d78e5ad17"
06:35:03.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:35:03 on dist-test-slave-jqnr"
06:35:03.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.168238 28514 fs_report.cc:383] FS layout report
06:35:03.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:35:03.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/wal
06:35:03.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/wal
06:35:03.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/data/data
06:35:03.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:35:03.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:35:03.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:35:03.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:35:03.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:35:03.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:35:03.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:35:03.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:35:03.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:35:03.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:35:03.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:35:03.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.216274 28514 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:35:03.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.222564 28514 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:35:03.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.223083 28514 kserver.cc:163] Server-wide thread pool size limit: 3276
06:35:03.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.304198 28514 rpc_server.cc:307] RPC server started. Bound to: 127.27.210.62:34959
06:35:03.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.304260 28584 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.210.62:34959 every 8 connection(s)
06:35:03.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.306825 28514 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/data/info.pb
06:35:03.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.313602 28585 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:35:03.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.315181 28488 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu as pid 28514
06:35:03.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.315591 28488 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/wal/instance
06:35:03.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.325615 28585 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.210.62" port: 34959 } has no permanent_uuid. Determining permanent_uuid...
06:35:03.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.27.210.61@KRBTEST.COM; defaulting to no policy
06:35:03.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:35:03.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.27.210.61@KRBTEST.COM" created.
06:35:03.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:03 dist-test-slave-jqnr krb5kdc[28501](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790663703, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.210.62@KRBTEST.COM for kudu/127.27.210.62@KRBTEST.COM
06:35:03.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:35:03.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.210.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kudu_127.27.210.61.keytab.
06:35:03.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.210.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kudu_127.27.210.61.keytab.
06:35:03.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.365948 28488 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:35:03.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:35:03.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/wal
06:35:03.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/data
06:35:03.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:35:03.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:35:03.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:35:03.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:35:03.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:35:03.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:35:03.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/logs
06:35:03.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/data/info.pb
06:35:03.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:35:03.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.210.61:44953
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.210.61
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kudu_127.27.210.61.keytab
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.210.61
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.210.20:43351
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.210.62:34959,127.27.210.61:44953,127.27.210.60:45231
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39537/jwks.json
06:35:03.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:35:03.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.368582 28585 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } has no permanent_uuid. Determining permanent_uuid...
06:35:03.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:03.371083 28536 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.210.61:44953: connect: Connection refused (error 111)
06:35:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:03.375245 28585 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.210.61:44953: Network error: Client connection negotiation failed: client connection to 127.27.210.61:44953: connect: Connection refused (error 111)
06:35:03.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.424739 28585 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } attempt: 1
06:35:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:03.428534 28585 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.210.61:44953: Network error: Client connection negotiation failed: client connection to 127.27.210.61:44953: connect: Connection refused (error 111)
06:35:03.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.496927 28585 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } attempt: 2
06:35:03.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:03.501055 28585 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.210.61:44953: Network error: Client connection negotiation failed: client connection to 127.27.210.61:44953: connect: Connection refused (error 111)
06:35:03.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.592558 28585 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } attempt: 3
06:35:03.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:03.596374 28585 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.210.61:44953: Network error: Client connection negotiation failed: client connection to 127.27.210.61:44953: connect: Connection refused (error 111)
06:35:03.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:03.696657 28591 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:35:03.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:03.697189 28591 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:35:03.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:03.697757 28591 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:35:03.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:03.733472 28591 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:35:03.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:03.733788 28591 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:35:03.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:03.734061 28591 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:35:03.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:03.734254 28591 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:35:03.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:03.734526 28591 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:35:03.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:03.734812 28591 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:35:03.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:03.734972 28591 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39537/jwks.json
06:35:03.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.739823 28585 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } attempt: 4
06:35:03.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:03.744613 28585 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.210.61:44953: Network error: Client connection negotiation failed: client connection to 127.27.210.61:44953: connect: Connection refused (error 111)
06:35:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.774972 28591 master_runner.cc:386] Master server non-default flags:
06:35:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:35:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.210.20:43351
06:35:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:35:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:35:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/data
06:35:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/wal
06:35:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:35:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:35:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.210.62:34959,127.27.210.61:44953,127.27.210.60:45231
06:35:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kudu_127.27.210.61.keytab
06:35:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.210.61
06:35:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:35:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:35:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:35:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.210.61:44953
06:35:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:35:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:35:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39537/jwks.json
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/data/info.pb
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.210.61
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/logs
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:35:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.776161 28591 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:35:03.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.777483 28591 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:35:03.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:03.796254 28599 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:35:04.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:04.044044 28585 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } attempt: 5
06:35:04.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:04.047708 28585 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.210.61:44953: Network error: Client connection negotiation failed: client connection to 127.27.210.61:44953: connect: Connection refused (error 111)
06:35:04.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:04.595304 28585 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } attempt: 6
06:35:04.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:04.602984 28585 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.210.61:44953: Network error: Client connection negotiation failed: client connection to 127.27.210.61:44953: connect: Connection refused (error 111)
06:35:05.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:03.799870 28591 server_base.cc:1034] running on GCE node
06:35:05.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:03.798069 28600 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:35:05.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:03.796267 28602 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:35:05.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.037438 28591 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:35:05.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.040628 28591 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:35:05.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.042261 28591 hybrid_clock.cc:648] HybridClock initialized: now 1790663705042227 us; error 41 us; skew 500 ppm
06:35:05.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.043057 28591 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:35:05.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:05 dist-test-slave-jqnr krb5kdc[28501](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790663705, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.210.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:35:05.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.053316 28591 init.cc:377] Logged in from keytab as kudu/127.27.210.61@KRBTEST.COM (short username kudu)
06:35:05.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.059676 28591 webserver.cc:460] Webserver started at http://127.27.210.61:35249/ using document root <none> and password file <none>
06:35:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.060688 28591 fs_manager.cc:359] Metadata directory not provided
06:35:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.060931 28591 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:35:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.061544 28591 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:35:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.067101 28591 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/data/instance:
06:35:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "19b1c7f7cd02496591ba26c8055f16e0"
06:35:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:35:05 on dist-test-slave-jqnr"
06:35:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.068284 28591 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/wal/instance:
06:35:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "19b1c7f7cd02496591ba26c8055f16e0"
06:35:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:35:05 on dist-test-slave-jqnr"
06:35:05.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.075560 28591 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
06:35:05.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.080904 28613 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:35:05.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.081828 28591 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
06:35:05.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.082190 28591 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/data,/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/wal
06:35:05.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "19b1c7f7cd02496591ba26c8055f16e0"
06:35:05.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:35:05 on dist-test-slave-jqnr"
06:35:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.082525 28591 fs_report.cc:383] FS layout report
06:35:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:35:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/wal
06:35:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/wal
06:35:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/data/data
06:35:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:35:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:35:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:35:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:35:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:35:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:35:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:35:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:35:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:35:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:35:05.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:35:05.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.118527 28591 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:35:05.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.124603 28591 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:35:05.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.125110 28591 kserver.cc:163] Server-wide thread pool size limit: 3276
06:35:05.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.204643 28591 rpc_server.cc:307] RPC server started. Bound to: 127.27.210.61:44953
06:35:05.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.204777 28664 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.210.61:44953 every 8 connection(s)
06:35:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.208160 28591 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/data/info.pb
06:35:05.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.213610 28665 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:35:05.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.215077 28488 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu as pid 28591
06:35:05.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.215392 28488 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/wal/instance
06:35:05.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.228850 28665 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.210.62" port: 34959 } has no permanent_uuid. Determining permanent_uuid...
06:35:05.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.27.210.60@KRBTEST.COM; defaulting to no policy
06:35:05.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:35:05.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.27.210.60@KRBTEST.COM" created.
06:35:05.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:05 dist-test-slave-jqnr krb5kdc[28501](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790663705, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.210.61@KRBTEST.COM for kudu/127.27.210.62@KRBTEST.COM
06:35:05.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
06:35:05.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.210.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kudu_127.27.210.60.keytab.
06:35:05.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.210.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kudu_127.27.210.60.keytab.
06:35:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.265529 28488 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:35:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:35:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/wal
06:35:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/data
06:35:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:35:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:35:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:35:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:35:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:35:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:35:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/logs
06:35:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/data/info.pb
06:35:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:35:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:35:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:35:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:35:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:35:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:35:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:35:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:35:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:35:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:35:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:35:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:35:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.210.60:45231
06:35:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.210.60
06:35:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:35:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kudu_127.27.210.60.keytab
06:35:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.210.60
06:35:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:35:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:35:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:35:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.210.20:43351
06:35:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:35:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:35:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:35:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:35:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.210.62:34959,127.27.210.61:44953,127.27.210.60:45231
06:35:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:35:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39537/jwks.json
06:35:05.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:35:05.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.272984 28665 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } has no permanent_uuid. Determining permanent_uuid...
06:35:05.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:05 dist-test-slave-jqnr krb5kdc[28501](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790663705, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.210.61@KRBTEST.COM for kudu/127.27.210.61@KRBTEST.COM
06:35:05.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.303218 28665 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } has no permanent_uuid. Determining permanent_uuid...
06:35:05.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.305235 28616 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.210.60:45231: connect: Connection refused (error 111)
06:35:05.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.309407 28665 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.210.60:45231: Network error: Client connection negotiation failed: client connection to 127.27.210.60:45231: connect: Connection refused (error 111)
06:35:05.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.358858 28665 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } attempt: 1
06:35:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.362777 28665 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.210.60:45231: Network error: Client connection negotiation failed: client connection to 127.27.210.60:45231: connect: Connection refused (error 111)
06:35:05.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.431131 28665 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } attempt: 2
06:35:05.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.435309 28665 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.210.60:45231: Network error: Client connection negotiation failed: client connection to 127.27.210.60:45231: connect: Connection refused (error 111)
06:35:05.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.526685 28665 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } attempt: 3
06:35:05.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.530515 28665 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.210.60:45231: Network error: Client connection negotiation failed: client connection to 127.27.210.60:45231: connect: Connection refused (error 111)
06:35:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.599731 28671 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:35:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.600449 28671 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:35:05.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.601220 28671 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:35:05.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.632321 28671 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:35:05.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.632634 28671 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:35:05.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.633023 28671 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:35:05.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.633352 28671 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:35:05.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.633663 28671 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:35:05.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.633965 28671 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:35:05.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.634188 28671 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39537/jwks.json
06:35:05.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.663579 28585 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } attempt: 7
06:35:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.673626 28671 master_runner.cc:386] Master server non-default flags:
06:35:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:35:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.210.20:43351
06:35:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:35:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:35:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/data
06:35:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/wal
06:35:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:35:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:35:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.210.62:34959,127.27.210.61:44953,127.27.210.60:45231
06:35:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kudu_127.27.210.60.keytab
06:35:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.210.60
06:35:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:35:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:35:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:35:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.210.60:45231
06:35:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:35:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:35:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:35:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39537/jwks.json
06:35:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:35:05.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:35:05.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:35:05.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:35:05.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:35:05.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/data/info.pb
06:35:05.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.210.60
06:35:05.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:35:05.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:35:05.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:35:05.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:35:05.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:35:05.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:35:05.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/logs
06:35:05.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:35:05.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:35:05.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:35:05.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:35:05.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:35:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:35:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:35:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:35:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:35:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:35:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.673933 28665 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } attempt: 4
06:35:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.675087 28671 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:35:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.676429 28671 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:35:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:05 dist-test-slave-jqnr krb5kdc[28501](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790663703, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.210.62@KRBTEST.COM for kudu/127.27.210.61@KRBTEST.COM
06:35:05.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.678812 28665 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.210.60:45231: Network error: Client connection negotiation failed: client connection to 127.27.210.60:45231: connect: Connection refused (error 111)
06:35:05.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.694710 28585 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } has no permanent_uuid. Determining permanent_uuid...
06:35:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.699633 28684 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:35:05.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.703850 28585 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.210.60:45231: Network error: Client connection negotiation failed: client connection to 127.27.210.60:45231: connect: Connection refused (error 111)
06:35:05.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.762282 28585 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } attempt: 1
06:35:05.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.765722 28585 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.210.60:45231: Network error: Client connection negotiation failed: client connection to 127.27.210.60:45231: connect: Connection refused (error 111)
06:35:05.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.847441 28585 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } attempt: 2
06:35:05.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.852404 28585 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.210.60:45231: Network error: Client connection negotiation failed: client connection to 127.27.210.60:45231: connect: Connection refused (error 111)
06:35:05.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.938086 28585 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } attempt: 3
06:35:05.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.942236 28585 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.210.60:45231: Network error: Client connection negotiation failed: client connection to 127.27.210.60:45231: connect: Connection refused (error 111)
06:35:05.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.978268 28665 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } attempt: 5
06:35:05.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.982967 28665 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.210.60:45231: Network error: Client connection negotiation failed: client connection to 127.27.210.60:45231: connect: Connection refused (error 111)
06:35:06.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:06.082895 28585 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } attempt: 4
06:35:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:06.088212 28585 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.210.60:45231: Network error: Client connection negotiation failed: client connection to 127.27.210.60:45231: connect: Connection refused (error 111)
06:35:06.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:06.371757 28585 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } attempt: 5
06:35:06.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:06.375212 28585 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.210.60:45231: Network error: Client connection negotiation failed: client connection to 127.27.210.60:45231: connect: Connection refused (error 111)
06:35:06.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:06.530359 28665 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } attempt: 6
06:35:06.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:06.539696 28665 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.210.60:45231: Network error: Client connection negotiation failed: client connection to 127.27.210.60:45231: connect: Connection refused (error 111)
06:35:06.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:06.927816 28585 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } attempt: 6
06:35:06.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:06.934973 28585 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.210.60:45231: Network error: Client connection negotiation failed: client connection to 127.27.210.60:45231: connect: Connection refused (error 111)
06:35:06.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.699723 28681 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:35:06.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:05.699741 28682 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:35:06.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:05.701885 28671 server_base.cc:1034] running on GCE node
06:35:06.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:06.966655 28671 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:35:06.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:06.969735 28671 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:35:06.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:06.971325 28671 hybrid_clock.cc:648] HybridClock initialized: now 1790663706971289 us; error 42 us; skew 500 ppm
06:35:06.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:06.972100 28671 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:35:06.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:06 dist-test-slave-jqnr krb5kdc[28501](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790663706, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.210.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:35:06.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:06.977780 28671 init.cc:377] Logged in from keytab as kudu/127.27.210.60@KRBTEST.COM (short username kudu)
06:35:06.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:06.983289 28671 webserver.cc:460] Webserver started at http://127.27.210.60:43631/ using document root <none> and password file <none>
06:35:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:06.984130 28671 fs_manager.cc:359] Metadata directory not provided
06:35:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:06.984306 28671 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:35:06.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:06.984948 28671 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:35:06.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:06.990180 28671 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/data/instance:
06:35:06.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "11fb2ea3421a4198a1f721b2f56be819"
06:35:06.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:35:06 on dist-test-slave-jqnr"
06:35:06.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:06.991354 28671 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/wal/instance:
06:35:06.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "11fb2ea3421a4198a1f721b2f56be819"
06:35:06.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:35:06 on dist-test-slave-jqnr"
06:35:07.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:06.999806 28671 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.004s
06:35:07.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.007735 28698 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:35:07.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.009012 28671 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
06:35:07.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.009405 28671 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/data,/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/wal
06:35:07.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "11fb2ea3421a4198a1f721b2f56be819"
06:35:07.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:35:06 on dist-test-slave-jqnr"
06:35:07.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.009791 28671 fs_report.cc:383] FS layout report
06:35:07.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:35:07.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/wal
06:35:07.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/wal
06:35:07.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/data/data
06:35:07.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:35:07.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:35:07.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:35:07.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:35:07.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:35:07.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:35:07.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:35:07.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:35:07.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:35:07.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:35:07.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:35:07.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.083674 28671 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:35:07.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.089839 28671 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:35:07.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.090417 28671 kserver.cc:163] Server-wide thread pool size limit: 3276
06:35:07.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.165805 28671 rpc_server.cc:307] RPC server started. Bound to: 127.27.210.60:45231
06:35:07.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.165851 28750 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.210.60:45231 every 8 connection(s)
06:35:07.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.168679 28671 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/data/info.pb
06:35:07.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.171909 28488 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu as pid 28671
06:35:07.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.172426 28488 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/wal/instance
06:35:07.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.175607 28751 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:35:07.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.191197 28751 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.210.62" port: 34959 } has no permanent_uuid. Determining permanent_uuid...
06:35:07.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:07 dist-test-slave-jqnr krb5kdc[28501](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790663701, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.27.210.60@KRBTEST.COM
06:35:07.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:07 dist-test-slave-jqnr krb5kdc[28501](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790663706, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.210.60@KRBTEST.COM for kudu/127.27.210.62@KRBTEST.COM
06:35:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.229311 28751 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } has no permanent_uuid. Determining permanent_uuid...
06:35:07.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:07 dist-test-slave-jqnr krb5kdc[28501](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790663706, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.210.60@KRBTEST.COM for kudu/127.27.210.61@KRBTEST.COM
06:35:07.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:07 dist-test-slave-jqnr krb5kdc[28501](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790663701, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.27.210.61@KRBTEST.COM
06:35:07.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.266512 28751 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } has no permanent_uuid. Determining permanent_uuid...
06:35:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:07 dist-test-slave-jqnr krb5kdc[28501](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790663706, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.210.60@KRBTEST.COM for kudu/127.27.210.60@KRBTEST.COM
06:35:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:07 dist-test-slave-jqnr krb5kdc[28501](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790663701, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.27.210.62@KRBTEST.COM
06:35:07.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.296288 28751 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: Bootstrap starting.
06:35:07.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.296638 28488 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
06:35:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.303583 28751 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: Neither blocks nor log segments found. Creating new log.
06:35:07.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.305711 28751 log.cc:824] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: Log is configured to *not* fsync() on all Append() calls
06:35:07.307 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:35:07.309 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
06:35:07.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.311158 28751 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: No bootstrap required, opened a new log
06:35:07.323 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
06:35:07.324 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
06:35:07.327 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.27.210.62"
port: 34959
}
bound_http_address {
host: "127.27.210.62"
port: 45111
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.27.210.61"
port: 44953
}
bound_http_address {
host: "127.27.210.61"
port: 35249
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.27.210.60"
port: 45231
}
bound_http_address {
host: "127.27.210.60"
port: 43631
}
}
}
06:35:07.328 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
06:35:07.328 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:35:07.328 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
06:35:07.330 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
06:35:07.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.338876 28751 raft_consensus.cc:348] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "912cbf41d7554146b0bd563d78e5ad17" member_type: VOTER last_known_addr { host: "127.27.210.62" port: 34959 } } peers { permanent_uuid: "19b1c7f7cd02496591ba26c8055f16e0" member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } } peers { permanent_uuid: "11fb2ea3421a4198a1f721b2f56be819" member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } }
06:35:07.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.339592 28751 raft_consensus.cc:374] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:35:07.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.339772 28751 raft_consensus.cc:729] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 11fb2ea3421a4198a1f721b2f56be819, State: Initialized, Role: FOLLOWER
06:35:07.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.340535 28751 consensus_queue.cc:260] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "912cbf41d7554146b0bd563d78e5ad17" member_type: VOTER last_known_addr { host: "127.27.210.62" port: 34959 } } peers { permanent_uuid: "19b1c7f7cd02496591ba26c8055f16e0" member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } } peers { permanent_uuid: "11fb2ea3421a4198a1f721b2f56be819" member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } }
06:35:07.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.343719 28761 sys_catalog.cc:455] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "912cbf41d7554146b0bd563d78e5ad17" member_type: VOTER last_known_addr { host: "127.27.210.62" port: 34959 } } peers { permanent_uuid: "19b1c7f7cd02496591ba26c8055f16e0" member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } } peers { permanent_uuid: "11fb2ea3421a4198a1f721b2f56be819" member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } } }
06:35:07.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.344606 28761 sys_catalog.cc:458] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [sys.catalog]: This master's current role is: FOLLOWER
06:35:07.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:07.345584 28705 tablet.cc:2369] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:35:07.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.345631 28751 sys_catalog.cc:564] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [sys.catalog]: configured and running, proceeding with master startup.
06:35:07.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:07 dist-test-slave-jqnr krb5kdc[28501](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790663707, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:35:07.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
06:35:07.370 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:35:07.372 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/krb5cc
06:35:07.380 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
06:35:07.381 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/krb5cc
06:35:07.382 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
06:35:07.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:07.381531 28773 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:07.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:07.383034 28773 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:07.386 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
06:35:07.388 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
06:35:07.388 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
06:35:07.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.374121 28671 master_runner.cc:186] Error getting master registration for 127.27.210.62:34959: OK, instance_id { permanent_uuid: "912cbf41d7554146b0bd563d78e5ad17" instance_seqno: 1790663703282985 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:35:07.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.411530 28671 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:35:07.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:07 dist-test-slave-jqnr krb5kdc[28501](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790663707, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:35:07.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
06:35:07.426 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:35:07.427 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/krb5cc
06:35:07.428 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
06:35:07.428 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.27.210.60:45231
06:35:07.430 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
06:35:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.430907 28488 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu with pid 28671
06:35:07.456 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:35:07.456 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.27.210.61:44953
06:35:07.457 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
06:35:07.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.457489 28488 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu with pid 28591
06:35:07.477 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:35:07.477 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.27.210.62:34959
06:35:07.478 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
06:35:07.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.478423 28488 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu with pid 28514
06:35:07.501 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:35:07.501 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.27.210.60:45231
06:35:07.503 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
06:35:07.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.505313 28488 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:35:07.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:35:07.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/wal
06:35:07.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/data
06:35:07.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:35:07.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:35:07.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:35:07.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:35:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:35:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:35:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/logs
06:35:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/data/info.pb
06:35:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:35:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:35:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:35:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:35:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:35:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:35:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:35:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:35:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:35:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:35:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:35:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:35:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.210.60:45231
06:35:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.210.60
06:35:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43631
06:35:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kudu_127.27.210.60.keytab
06:35:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.210.60
06:35:07.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:35:07.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:35:07.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:35:07.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.210.20:43351
06:35:07.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:35:07.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:35:07.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:35:07.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:35:07.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.210.62:34959,127.27.210.61:44953,127.27.210.60:45231
06:35:07.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:35:07.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39537/jwks.json
06:35:07.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:35:07.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:07.811241 28777 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:35:07.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:07.811926 28777 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:35:07.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:07.812803 28777 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:35:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:07.846395 28777 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:35:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:07.846700 28777 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:35:07.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:07.847097 28777 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:35:07.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:07.847422 28777 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:35:07.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:07.847738 28777 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:35:07.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:07.848045 28777 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:35:07.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:07.848266 28777 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39537/jwks.json
06:35:07.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.885732 28777 master_runner.cc:386] Master server non-default flags:
06:35:07.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:35:07.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.210.20:43351
06:35:07.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:35:07.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:35:07.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/data
06:35:07.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/wal
06:35:07.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:35:07.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:35:07.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.210.62:34959,127.27.210.61:44953,127.27.210.60:45231
06:35:07.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kudu_127.27.210.60.keytab
06:35:07.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.210.60
06:35:07.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:35:07.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:35:07.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:35:07.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.210.60:45231
06:35:07.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:35:07.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:35:07.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:35:07.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39537/jwks.json
06:35:07.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:35:07.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:35:07.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:35:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:35:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:35:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/data/info.pb
06:35:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.210.60
06:35:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43631
06:35:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:35:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:35:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:35:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:35:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:35:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/logs
06:35:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:35:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:35:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:35:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:35:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:35:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:35:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:35:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:35:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:35:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:35:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.887284 28777 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:35:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.888628 28777 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:35:07.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:07.906354 28785 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:35:09.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:07.906442 28787 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:35:09.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:07.908814 28777 server_base.cc:1034] running on GCE node
06:35:09.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:07.906349 28784 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:35:09.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.141415 28777 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:35:09.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.144671 28777 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:35:09.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.146092 28777 hybrid_clock.cc:648] HybridClock initialized: now 1790663709146070 us; error 57 us; skew 500 ppm
06:35:09.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.146844 28777 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:35:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:09 dist-test-slave-jqnr krb5kdc[28501](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790663709, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.210.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:35:09.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.159595 28777 init.cc:377] Logged in from keytab as kudu/127.27.210.60@KRBTEST.COM (short username kudu)
06:35:09.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.165249 28777 webserver.cc:460] Webserver started at http://127.27.210.60:43631/ using document root <none> and password file <none>
06:35:09.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.166173 28777 fs_manager.cc:359] Metadata directory not provided
06:35:09.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.166368 28777 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:35:09.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.173885 28777 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
06:35:09.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.178313 28795 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:35:09.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.179525 28777 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
06:35:09.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.179797 28777 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/data,/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/wal
06:35:09.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "11fb2ea3421a4198a1f721b2f56be819"
06:35:09.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:35:06 on dist-test-slave-jqnr"
06:35:09.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.182173 28777 fs_report.cc:383] FS layout report
06:35:09.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:35:09.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/wal
06:35:09.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/wal
06:35:09.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/data/data
06:35:09.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:35:09.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:35:09.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:35:09.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:35:09.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:35:09.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:35:09.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:35:09.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:35:09.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:35:09.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:35:09.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:35:09.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.218019 28777 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:35:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.224747 28777 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:35:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.225152 28777 kserver.cc:163] Server-wide thread pool size limit: 3276
06:35:09.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.304435 28777 rpc_server.cc:307] RPC server started. Bound to: 127.27.210.60:45231
06:35:09.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.304515 28846 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.210.60:45231 every 8 connection(s)
06:35:09.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.308770 28777 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-2/data/info.pb
06:35:09.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.317195 28488 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu as pid 28777
06:35:09.318 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:35:09.318 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.27.210.61:44953
06:35:09.318 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
06:35:09.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.320067 28488 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:35:09.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:35:09.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/wal
06:35:09.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/data
06:35:09.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:35:09.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:35:09.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:35:09.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:35:09.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:35:09.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:35:09.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/logs
06:35:09.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/data/info.pb
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.210.61:44953
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.210.61
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35249
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kudu_127.27.210.61.keytab
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.210.61
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.210.20:43351
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.210.62:34959,127.27.210.61:44953,127.27.210.60:45231
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39537/jwks.json
06:35:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:35:09.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.322870 28847 sys_catalog.cc:263] Verifying existing consensus state
06:35:09.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.331039 28847 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: Bootstrap starting.
06:35:09.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.372568 28847 log.cc:824] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: Log is configured to *not* fsync() on all Append() calls
06:35:09.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.377454 28847 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:35:09.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.378459 28847 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: Bootstrap complete.
06:35:09.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.407299 28847 raft_consensus.cc:348] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "912cbf41d7554146b0bd563d78e5ad17" member_type: VOTER last_known_addr { host: "127.27.210.62" port: 34959 } } peers { permanent_uuid: "19b1c7f7cd02496591ba26c8055f16e0" member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } } peers { permanent_uuid: "11fb2ea3421a4198a1f721b2f56be819" member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } }
06:35:09.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.409660 28847 raft_consensus.cc:374] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:35:09.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.409941 28847 raft_consensus.cc:729] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 11fb2ea3421a4198a1f721b2f56be819, State: Initialized, Role: FOLLOWER
06:35:09.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.410740 28847 consensus_queue.cc:260] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "912cbf41d7554146b0bd563d78e5ad17" member_type: VOTER last_known_addr { host: "127.27.210.62" port: 34959 } } peers { permanent_uuid: "19b1c7f7cd02496591ba26c8055f16e0" member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } } peers { permanent_uuid: "11fb2ea3421a4198a1f721b2f56be819" member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } }
06:35:09.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.413386 28853 sys_catalog.cc:455] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "912cbf41d7554146b0bd563d78e5ad17" member_type: VOTER last_known_addr { host: "127.27.210.62" port: 34959 } } peers { permanent_uuid: "19b1c7f7cd02496591ba26c8055f16e0" member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } } peers { permanent_uuid: "11fb2ea3421a4198a1f721b2f56be819" member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } } }
06:35:09.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.414294 28853 sys_catalog.cc:458] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [sys.catalog]: This master's current role is: FOLLOWER
06:35:09.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.414891 28847 sys_catalog.cc:564] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [sys.catalog]: configured and running, proceeding with master startup.
06:35:09.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:09.447197 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:09.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:09.447570 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:09.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.438966 28777 master_runner.cc:186] Error getting master registration for 127.27.210.62:34959: Network error: Client connection negotiation failed: client connection to 127.27.210.62:34959: connect: Connection refused (error 111),
06:35:09.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.470742 28777 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:35:09.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:09.480036 28801 tablet.cc:2369] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:35:09.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:09.681404 28849 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:35:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:09.681916 28849 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:35:09.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:09.682525 28849 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:35:09.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:09.715730 28849 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:35:09.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:09.716010 28849 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:35:09.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:09.716355 28849 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:35:09.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:09.716694 28849 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:35:09.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:09.716897 28849 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:35:09.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:09.717206 28849 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:35:09.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:09.717428 28849 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39537/jwks.json
06:35:09.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.760131 28849 master_runner.cc:386] Master server non-default flags:
06:35:09.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:35:09.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.210.20:43351
06:35:09.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:35:09.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:35:09.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/data
06:35:09.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/wal
06:35:09.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:35:09.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:35:09.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.210.62:34959,127.27.210.61:44953,127.27.210.60:45231
06:35:09.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kudu_127.27.210.61.keytab
06:35:09.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.210.61
06:35:09.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:35:09.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:35:09.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.210.61:44953
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39537/jwks.json
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/data/info.pb
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.210.61
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35249
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/logs
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:35:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:35:09.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:35:09.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:35:09.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:35:09.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.761684 28849 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:35:09.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.763025 28849 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:35:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:09.783264 28872 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:35:09.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.857429 28853 raft_consensus.cc:482] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:35:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.858250 28853 raft_consensus.cc:504] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "912cbf41d7554146b0bd563d78e5ad17" member_type: VOTER last_known_addr { host: "127.27.210.62" port: 34959 } } peers { permanent_uuid: "19b1c7f7cd02496591ba26c8055f16e0" member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } } peers { permanent_uuid: "11fb2ea3421a4198a1f721b2f56be819" member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } }
06:35:09.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:09.864264 28796 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.210.62:34959: connect: Connection refused (error 111)
06:35:09.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.867791 28853 leader_election.cc:288] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 912cbf41d7554146b0bd563d78e5ad17 (127.27.210.62:34959), 19b1c7f7cd02496591ba26c8055f16e0 (127.27.210.61:44953)
06:35:09.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:09.872294 28798 leader_election.cc:334] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 19b1c7f7cd02496591ba26c8055f16e0 (127.27.210.61:44953): Network error: Client connection negotiation failed: client connection to 127.27.210.61:44953: connect: Connection refused (error 111)
06:35:09.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:09.873049 28796 leader_election.cc:334] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 912cbf41d7554146b0bd563d78e5ad17 (127.27.210.62:34959): Network error: Client connection negotiation failed: client connection to 127.27.210.62:34959: connect: Connection refused (error 111)
06:35:09.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.873454 28796 leader_election.cc:302] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 11fb2ea3421a4198a1f721b2f56be819; no voters: 19b1c7f7cd02496591ba26c8055f16e0, 912cbf41d7554146b0bd563d78e5ad17
06:35:09.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:09.874213 28853 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:35:10.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:10.454120 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:10.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:10.454639 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:11.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:09.783582 28873 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:35:11.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:09.784902 28875 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:35:11.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:11.120735 28874 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
06:35:11.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.121347 28849 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
06:35:11.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.125993 28849 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:35:11.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.129333 28849 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:35:11.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.130805 28849 hybrid_clock.cc:648] HybridClock initialized: now 1790663711130713 us; error 104 us; skew 500 ppm
06:35:11.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.131748 28849 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:35:11.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:11 dist-test-slave-jqnr krb5kdc[28501](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790663711, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.210.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:35:11.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.145154 28849 init.cc:377] Logged in from keytab as kudu/127.27.210.61@KRBTEST.COM (short username kudu)
06:35:11.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.151983 28849 webserver.cc:460] Webserver started at http://127.27.210.61:35249/ using document root <none> and password file <none>
06:35:11.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.152999 28849 fs_manager.cc:359] Metadata directory not provided
06:35:11.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.153281 28849 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:35:11.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.162776 28849 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.005s sys 0.000s
06:35:11.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.167922 28884 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:35:11.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.169396 28849 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
06:35:11.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.169797 28849 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/data,/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/wal
06:35:11.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "19b1c7f7cd02496591ba26c8055f16e0"
06:35:11.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:35:05 on dist-test-slave-jqnr"
06:35:11.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.173066 28849 fs_report.cc:383] FS layout report
06:35:11.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:35:11.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/wal
06:35:11.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/wal
06:35:11.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/data/data
06:35:11.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:35:11.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:35:11.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:35:11.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:35:11.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:35:11.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:35:11.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:35:11.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:35:11.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:35:11.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:35:11.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:35:11.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.226145 28849 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:35:11.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.234321 28849 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:35:11.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.234988 28849 kserver.cc:163] Server-wide thread pool size limit: 3276
06:35:11.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.344285 28849 rpc_server.cc:307] RPC server started. Bound to: 127.27.210.61:44953
06:35:11.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.344358 28935 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.210.61:44953 every 8 connection(s)
06:35:11.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.347138 28849 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/data/info.pb
06:35:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.357111 28488 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu as pid 28849
06:35:11.358 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:35:11.358 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.27.210.62:34959
06:35:11.359 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
06:35:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.360787 28936 sys_catalog.cc:263] Verifying existing consensus state
06:35:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.360786 28488 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:35:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu
06:35:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/wal
06:35:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/data
06:35:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:35:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:35:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:35:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:35:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:35:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:35:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/logs
06:35:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/data/info.pb
06:35:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:35:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:35:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.210.62:34959
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.210.62
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45111
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kudu_127.27.210.62.keytab
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.210.62
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.210.20:43351
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.210.62:34959,127.27.210.61:44953,127.27.210.60:45231
06:35:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:35:11.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39537/jwks.json
06:35:11.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
06:35:11.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 06:35:11.361694 28936 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
06:35:11.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.362143 28849 master.cc:537] Master@127.27.210.61:44953 shutting down...
06:35:11.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.425284 28849 master.cc:552] Master@127.27.210.61:44953 shutdown complete.
06:35:11.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
06:35:11.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:11.459563 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:11.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:11.459967 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:11.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:11.680647 28938 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:35:11.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:11.681182 28938 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:35:11.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:11.681818 28938 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:35:11.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.712944 28943 raft_consensus.cc:482] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:35:11.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:11.713145 28938 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
06:35:11.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:11.713436 28938 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:35:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:11.713790 28938 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:35:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.713423 28943 raft_consensus.cc:504] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "912cbf41d7554146b0bd563d78e5ad17" member_type: VOTER last_known_addr { host: "127.27.210.62" port: 34959 } } peers { permanent_uuid: "19b1c7f7cd02496591ba26c8055f16e0" member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } } peers { permanent_uuid: "11fb2ea3421a4198a1f721b2f56be819" member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } }
06:35:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:11.714075 28938 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:35:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:11.714349 28938 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:35:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:11.714565 28938 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:35:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:11.714771 28938 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39537/jwks.json
06:35:11.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.715528 28943 leader_election.cc:288] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 912cbf41d7554146b0bd563d78e5ad17 (127.27.210.62:34959), 19b1c7f7cd02496591ba26c8055f16e0 (127.27.210.61:44953)
06:35:11.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:11.723045 28796 leader_election.cc:334] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 912cbf41d7554146b0bd563d78e5ad17 (127.27.210.62:34959): Network error: Client connection negotiation failed: client connection to 127.27.210.62:34959: connect: Connection refused (error 111)
06:35:11.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:11.723699 28798 leader_election.cc:334] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 19b1c7f7cd02496591ba26c8055f16e0 (127.27.210.61:44953): Network error: Client connection negotiation failed: client connection to 127.27.210.61:44953: connect: Connection refused (error 111)
06:35:11.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.724083 28798 leader_election.cc:302] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 11fb2ea3421a4198a1f721b2f56be819; no voters: 19b1c7f7cd02496591ba26c8055f16e0, 912cbf41d7554146b0bd563d78e5ad17
06:35:11.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.724833 28943 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:35:11.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.760149 28938 master_runner.cc:386] Master server non-default flags:
06:35:11.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:35:11.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.210.20:43351
06:35:11.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:35:11.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:35:11.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/data
06:35:11.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/wal
06:35:11.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
06:35:11.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:35:11.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.210.62:34959,127.27.210.61:44953,127.27.210.60:45231
06:35:11.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/krb5kdc/kudu_127.27.210.62.keytab
06:35:11.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.210.62
06:35:11.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:35:11.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:35:11.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:35:11.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.210.62:34959
06:35:11.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:35:11.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:35:11.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:35:11.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39537/jwks.json
06:35:11.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
06:35:11.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
06:35:11.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
06:35:11.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:35:11.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:35:11.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/data/info.pb
06:35:11.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.210.62
06:35:11.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45111
06:35:11.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:35:11.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:35:11.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:35:11.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:35:11.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:35:11.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/logs
06:35:11.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:35:11.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:35:11.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:35:11.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:35:11.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:35:11.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:35:11.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:35:11.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 05:43:14 UTC on ccycloud.quasar-iapthk.root.comops.site
06:35:11.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679255
06:35:11.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
06:35:11.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.761385 28938 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:35:11.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.762353 28938 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:35:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:11.781502 28949 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:35:11.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:11.964016 28192 debug-util.cc:398] Leaking SignalData structure 0x7b08000bc0a0 after lost signal to thread 28064
06:35:11.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:11.965379 28192 debug-util.cc:398] Leaking SignalData structure 0x7b08000bcb40 after lost signal to thread 28195
06:35:12.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:12.464335 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:12.465046 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:13.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:11.781960 28948 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:35:13.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:11.783471 28938 server_base.cc:1034] running on GCE node
06:35:13.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:11.782670 28951 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:35:13.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:13.073832 28938 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:35:13.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:13.077435 28938 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:35:13.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:13.079157 28938 hybrid_clock.cc:648] HybridClock initialized: now 1790663713079091 us; error 98 us; skew 500 ppm
06:35:13.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:13.080067 28938 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:35:13.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:13 dist-test-slave-jqnr krb5kdc[28501](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790663713, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.210.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:35:13.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:13.086026 28938 init.cc:377] Logged in from keytab as kudu/127.27.210.62@KRBTEST.COM (short username kudu)
06:35:13.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:13.092223 28938 webserver.cc:460] Webserver started at http://127.27.210.62:45111/ using document root <none> and password file <none>
06:35:13.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:13.093182 28938 fs_manager.cc:359] Metadata directory not provided
06:35:13.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:13.093439 28938 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:35:13.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:13.101588 28938 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.007s sys 0.001s
06:35:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:13.106319 28960 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:35:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:13.107488 28938 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
06:35:13.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:13.107821 28938 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/data,/tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/wal
06:35:13.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "912cbf41d7554146b0bd563d78e5ad17"
06:35:13.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 06:35:03 on dist-test-slave-jqnr"
06:35:13.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:13.110178 28938 fs_report.cc:383] FS layout report
06:35:13.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:35:13.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/wal
06:35:13.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/wal
06:35:13.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/data/data
06:35:13.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:35:13.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:35:13.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:35:13.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:35:13.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:35:13.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:35:13.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:35:13.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:35:13.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:35:13.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:35:13.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:35:13.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:13.149190 28938 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:35:13.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:13.156102 28938 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:35:13.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:13.156824 28938 kserver.cc:163] Server-wide thread pool size limit: 3276
06:35:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:13.243835 28938 rpc_server.cc:307] RPC server started. Bound to: 127.27.210.62:34959
06:35:13.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:13.244005 29011 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.210.62:34959 every 8 connection(s)
06:35:13.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:13.248255 28938 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/data/info.pb
06:35:13.251 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:35:13.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:13.250401 28488 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu as pid 28938
06:35:13.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:13.271518 29012 sys_catalog.cc:263] Verifying existing consensus state
06:35:13.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 06:35:13.272521 29012 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
06:35:13.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:13.272945 28938 master.cc:537] Master@127.27.210.62:34959 shutting down...
06:35:13.339 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.27.210.60:45231(127.27.210.60:45231)] connecting to peer
06:35:13.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:13.403286 28938 master.cc:552] Master@127.27.210.62:34959 shutdown complete.
06:35:13.412 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:13.415 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:13.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskYC5Dsr/test-tmp/mini-kudu-cluster6242693270504846078/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
06:35:13.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:13.469393 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:13.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:13.470003 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:13.485 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.27.210.60:45231(127.27.210.60:45231)] Successfully connected to peer
06:35:13.484 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:13.487 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:13.490 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:13.567 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:13.567 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:13.568 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:13.583 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
06:35:13.593 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.27.210.60
06:35:13.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:13 dist-test-slave-jqnr krb5kdc[28501](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790663707, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.27.210.60@KRBTEST.COM
06:35:13.892 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x03f6d071, L:/127.0.0.1:51238 - R:/127.27.210.60:45231] using SASL/GSSAPI
06:35:13.911 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:13.919 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:13.922 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:13.923 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:13.923 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:13.924 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:13.926 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:13.926 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:13.927 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:13.927 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:13.937 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:13.938 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:13.939 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:13.939 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:13.940 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:13.942 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:13.942 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:13.943 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:13.943 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:13.957 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:13.958 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:13.961 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:13.961 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:13.962 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:13.962 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:13.962 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:13.963 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:13.965 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:13.996 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:13.997 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:13.999 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:14.000 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:14.001 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:14.001 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:14.001 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:14.002 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:14.002 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:14.037 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:14.038 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:14.041 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:14.041 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:14.042 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:14.043 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:14.043 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:14.043 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:14.044 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:14.057 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:14.058 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:14.061 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:14.061 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:14.062 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:14.063 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:14.063 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:14.064 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:14.064 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:14.196 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:14.199 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:14.200 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:14.200 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:14.201 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:14.203 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:14.203 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:14.204 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:14.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:14.204464 29018 raft_consensus.cc:482] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:35:14.204 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:14.204926 29018 raft_consensus.cc:504] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "912cbf41d7554146b0bd563d78e5ad17" member_type: VOTER last_known_addr { host: "127.27.210.62" port: 34959 } } peers { permanent_uuid: "19b1c7f7cd02496591ba26c8055f16e0" member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } } peers { permanent_uuid: "11fb2ea3421a4198a1f721b2f56be819" member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } }
06:35:14.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:14.206696 29018 leader_election.cc:288] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 912cbf41d7554146b0bd563d78e5ad17 (127.27.210.62:34959), 19b1c7f7cd02496591ba26c8055f16e0 (127.27.210.61:44953)
06:35:14.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:14.212065 28798 leader_election.cc:334] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 19b1c7f7cd02496591ba26c8055f16e0 (127.27.210.61:44953): Network error: Client connection negotiation failed: client connection to 127.27.210.61:44953: connect: Connection refused (error 111)
06:35:14.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:14.212647 28796 leader_election.cc:334] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 912cbf41d7554146b0bd563d78e5ad17 (127.27.210.62:34959): Network error: Client connection negotiation failed: client connection to 127.27.210.62:34959: connect: Connection refused (error 111)
06:35:14.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:14.212946 28796 leader_election.cc:302] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 11fb2ea3421a4198a1f721b2f56be819; no voters: 19b1c7f7cd02496591ba26c8055f16e0, 912cbf41d7554146b0bd563d78e5ad17
06:35:14.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:14.213501 29018 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:35:14.317 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:14.319 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:14.319 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:14.320 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:14.320 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:14.322 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:14.322 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:14.323 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:14.323 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:14.472869 28777 master_runner.cc:186] Error getting master registration for 127.27.210.62:34959: Network error: Client connection negotiation failed: client connection to 127.27.210.62:34959: connect: Connection refused (error 111),
06:35:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:14.473354 28777 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:35:14.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:14.473824 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:14.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:14.474041 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:14.817 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:14.819 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:14.819 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:14.820 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:14.821 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:14.822 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:14.822 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:14.823 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:14.823 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:15.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:15.477147 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:15.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:15.477428 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:15.817 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:15.819 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:15.819 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:15.819 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:15.822 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:15.823 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:15.823 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:15.823 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:15.823 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:16.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:16.480533 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:16.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:16.480809 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:17.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:17.483608 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:17.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:17.483858 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:17.757 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:17.758 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:17.759 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:17.759 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:17.760 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:17.760 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:17.760 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:17.760 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:17.761 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:18.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:18.362957 29029 raft_consensus.cc:482] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:35:18.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:18.363540 29029 raft_consensus.cc:504] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "912cbf41d7554146b0bd563d78e5ad17" member_type: VOTER last_known_addr { host: "127.27.210.62" port: 34959 } } peers { permanent_uuid: "19b1c7f7cd02496591ba26c8055f16e0" member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } } peers { permanent_uuid: "11fb2ea3421a4198a1f721b2f56be819" member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } }
06:35:18.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:18.365736 29029 leader_election.cc:288] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 912cbf41d7554146b0bd563d78e5ad17 (127.27.210.62:34959), 19b1c7f7cd02496591ba26c8055f16e0 (127.27.210.61:44953)
06:35:18.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:18.368631 28796 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.210.62:34959: connect: Connection refused (error 111) [suppressed 6 similar messages]
06:35:18.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:18.371454 28798 leader_election.cc:334] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 19b1c7f7cd02496591ba26c8055f16e0 (127.27.210.61:44953): Network error: Client connection negotiation failed: client connection to 127.27.210.61:44953: connect: Connection refused (error 111)
06:35:18.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:18.371927 28796 leader_election.cc:334] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 912cbf41d7554146b0bd563d78e5ad17 (127.27.210.62:34959): Network error: Client connection negotiation failed: client connection to 127.27.210.62:34959: connect: Connection refused (error 111)
06:35:18.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:18.372257 28796 leader_election.cc:302] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 11fb2ea3421a4198a1f721b2f56be819; no voters: 19b1c7f7cd02496591ba26c8055f16e0, 912cbf41d7554146b0bd563d78e5ad17
06:35:18.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:18.373065 29029 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:35:18.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:18.487424 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:18.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:18.487756 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:19.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:19 dist-test-slave-jqnr krb5kdc[28501](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790663719, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.210.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:35:19.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:19.163334 28791 init.cc:295] Successfully reacquired a new kerberos TGT
06:35:19.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:19.476336 28777 master_runner.cc:186] Error getting master registration for 127.27.210.62:34959: Network error: Client connection negotiation failed: client connection to 127.27.210.62:34959: connect: Connection refused (error 111),
06:35:19.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:19.476825 28777 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:35:19.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:19.490628 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:19.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:19.490914 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:20.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:20.493788 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:20.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:20.494069 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:21.017 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:21.018 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:21.019 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:21.019 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:21.019 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:21.022 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:21.023 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:21.023 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:21.023 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:21.436 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:21.438 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:21.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:21.439 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:21.439 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:21.441 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:21.441 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:21.441 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:21.442 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:21.496984 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:21.497298 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:22.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:22.047217 29037 raft_consensus.cc:482] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:35:22.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:22.047560 29037 raft_consensus.cc:504] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "912cbf41d7554146b0bd563d78e5ad17" member_type: VOTER last_known_addr { host: "127.27.210.62" port: 34959 } } peers { permanent_uuid: "19b1c7f7cd02496591ba26c8055f16e0" member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } } peers { permanent_uuid: "11fb2ea3421a4198a1f721b2f56be819" member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } }
06:35:22.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:22.049099 29037 leader_election.cc:288] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 912cbf41d7554146b0bd563d78e5ad17 (127.27.210.62:34959), 19b1c7f7cd02496591ba26c8055f16e0 (127.27.210.61:44953)
06:35:22.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:22.053727 28796 leader_election.cc:334] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 912cbf41d7554146b0bd563d78e5ad17 (127.27.210.62:34959): Network error: Client connection negotiation failed: client connection to 127.27.210.62:34959: connect: Connection refused (error 111)
06:35:22.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:22.054157 28798 leader_election.cc:334] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 19b1c7f7cd02496591ba26c8055f16e0 (127.27.210.61:44953): Network error: Client connection negotiation failed: client connection to 127.27.210.61:44953: connect: Connection refused (error 111)
06:35:22.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:22.054507 28798 leader_election.cc:302] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 11fb2ea3421a4198a1f721b2f56be819; no voters: 19b1c7f7cd02496591ba26c8055f16e0, 912cbf41d7554146b0bd563d78e5ad17
06:35:22.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:22.055114 29037 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:35:22.197 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:22.198 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:22.199 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:22.199 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:22.200 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:22.202 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:22.202 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:22.203 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:22.203 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:22.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:22.500233 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:22.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:22.500592 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:23.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:23.503402 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:23.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:23.503713 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:24.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:24.479442 28777 master_runner.cc:186] Error getting master registration for 127.27.210.62:34959: Network error: Client connection negotiation failed: client connection to 127.27.210.62:34959: connect: Connection refused (error 111),
06:35:24.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:24.479804 28777 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:35:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:24.506417 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:24.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:24.506742 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:25.477 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:25.481 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:25.481 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:25.481 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:25.483 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:25.483 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:25.483 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:25.483 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:25.483 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:25.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:25.509508 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:25.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:25.509809 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:26.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:26.118801 29046 raft_consensus.cc:482] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:35:26.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:26.119280 29046 raft_consensus.cc:504] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "912cbf41d7554146b0bd563d78e5ad17" member_type: VOTER last_known_addr { host: "127.27.210.62" port: 34959 } } peers { permanent_uuid: "19b1c7f7cd02496591ba26c8055f16e0" member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } } peers { permanent_uuid: "11fb2ea3421a4198a1f721b2f56be819" member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } }
06:35:26.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:26.121163 29046 leader_election.cc:288] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 912cbf41d7554146b0bd563d78e5ad17 (127.27.210.62:34959), 19b1c7f7cd02496591ba26c8055f16e0 (127.27.210.61:44953)
06:35:26.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:26.123456 28796 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.210.62:34959: connect: Connection refused (error 111) [suppressed 2 similar messages]
06:35:26.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:26.125768 28798 leader_election.cc:334] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 19b1c7f7cd02496591ba26c8055f16e0 (127.27.210.61:44953): Network error: Client connection negotiation failed: client connection to 127.27.210.61:44953: connect: Connection refused (error 111)
06:35:26.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:26.126159 28796 leader_election.cc:334] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 912cbf41d7554146b0bd563d78e5ad17 (127.27.210.62:34959): Network error: Client connection negotiation failed: client connection to 127.27.210.62:34959: connect: Connection refused (error 111)
06:35:26.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:26.126443 28796 leader_election.cc:302] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 11fb2ea3421a4198a1f721b2f56be819; no voters: 19b1c7f7cd02496591ba26c8055f16e0, 912cbf41d7554146b0bd563d78e5ad17
06:35:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:26.126937 29046 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:35:26.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:26.512817 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:26.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:26.513175 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:27.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:27.516332 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:27.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:27.516644 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:28.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:28.520124 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:28.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:28.520584 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:29.037 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:29.039 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:29.040 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:29.040 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:29.040 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:29.042 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:29.042 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:29.042 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:29.043 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:29.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:29 dist-test-slave-jqnr krb5kdc[28501](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790663729, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.210.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:35:29.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:29.165740 28791 init.cc:295] Successfully reacquired a new kerberos TGT
06:35:29.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:29.482774 28777 master_runner.cc:186] Error getting master registration for 127.27.210.62:34959: Network error: Client connection negotiation failed: client connection to 127.27.210.62:34959: connect: Connection refused (error 111),
06:35:29.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:29.483160 28777 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:35:29.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:29.523856 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:29.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:29.524202 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:30.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:30.527062 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:30.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:30.527316 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:30.737 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:30.739 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:30.739 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:30.739 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:30.740 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:30.741 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:30.741 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:30.742 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:30.742 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:31.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:31.016323 28192 debug-util.cc:398] Leaking SignalData structure 0x7b08000bb840 after lost signal to thread 28064
06:35:31.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:31.017339 28192 debug-util.cc:398] Leaking SignalData structure 0x7b08000aebe0 after lost signal to thread 28195
06:35:31.217 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:31.218 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:31.219 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:31.219 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:31.220 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:31.221 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:31.221 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:31.221 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:31.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:31.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:31.530063 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:31.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:31.530315 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:32.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:32.533207 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:32.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:32.533541 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:33.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:33.536446 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:33.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:33.536731 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:33.537 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:33.538 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:33.540 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:33.540 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:33.540 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:33.540 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:33.540 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:33.541 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:33.542 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:34.457 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:34.458 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:34.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:34.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:34.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:34.458 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:34.459 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:34.459 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:34.460 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:34.485785 28777 master_runner.cc:186] Error getting master registration for 127.27.210.62:34959: Network error: Client connection negotiation failed: client connection to 127.27.210.62:34959: connect: Connection refused (error 111),
06:35:34.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:34.486114 28777 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:35:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:34.540694 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:34.541020 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:34.557 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:34.557 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:34.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:34.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:34.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:34.559 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:34.559 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:34.559 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:34.560 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:35.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:35.544953 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:35.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:35.545288 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:36.549428 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:36.549741 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:36.657 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:36.659 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:36.660 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:36.660 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:36.660 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:36.661 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:36.661 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:36.661 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:36.662 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:37.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:37.553566 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:37.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:37.553894 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:38.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:38.557816 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:38.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:38.558141 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:38.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:38.636128 29065 raft_consensus.cc:482] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:35:38.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:38.636639 29065 raft_consensus.cc:504] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "912cbf41d7554146b0bd563d78e5ad17" member_type: VOTER last_known_addr { host: "127.27.210.62" port: 34959 } } peers { permanent_uuid: "19b1c7f7cd02496591ba26c8055f16e0" member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } } peers { permanent_uuid: "11fb2ea3421a4198a1f721b2f56be819" member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } }
06:35:38.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:38.638314 29065 leader_election.cc:288] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 912cbf41d7554146b0bd563d78e5ad17 (127.27.210.62:34959), 19b1c7f7cd02496591ba26c8055f16e0 (127.27.210.61:44953)
06:35:38.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:38.640754 28796 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.210.62:34959: connect: Connection refused (error 111) [suppressed 1 similar messages]
06:35:38.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:38.644119 28796 leader_election.cc:334] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 912cbf41d7554146b0bd563d78e5ad17 (127.27.210.62:34959): Network error: Client connection negotiation failed: client connection to 127.27.210.62:34959: connect: Connection refused (error 111)
06:35:38.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:38.644930 28798 leader_election.cc:334] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 19b1c7f7cd02496591ba26c8055f16e0 (127.27.210.61:44953): Network error: Client connection negotiation failed: client connection to 127.27.210.61:44953: connect: Connection refused (error 111)
06:35:38.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:38.645213 28798 leader_election.cc:302] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 11fb2ea3421a4198a1f721b2f56be819; no voters: 19b1c7f7cd02496591ba26c8055f16e0, 912cbf41d7554146b0bd563d78e5ad17
06:35:38.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:38.645737 29065 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:35:39.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 06:35:39 dist-test-slave-jqnr krb5kdc[28501](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790663739, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.210.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
06:35:39.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:39.168344 28791 init.cc:295] Successfully reacquired a new kerberos TGT
06:35:39.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:39.489537 28777 master_runner.cc:186] Error getting master registration for 127.27.210.62:34959: Network error: Client connection negotiation failed: client connection to 127.27.210.62:34959: connect: Connection refused (error 111),
06:35:39.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:39.490010 28777 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:35:39.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:39.560871 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:39.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:39.561149 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:40.237 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:40.237 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:40.238 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:40.238 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:40.238 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:40.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:40.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:40.239 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:40.241 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:40.563784 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:40.564071 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:41.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:41.567054 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:41.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:41.567338 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:41.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:41.890138 29073 raft_consensus.cc:482] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:35:41.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:41.890614 29073 raft_consensus.cc:504] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "912cbf41d7554146b0bd563d78e5ad17" member_type: VOTER last_known_addr { host: "127.27.210.62" port: 34959 } } peers { permanent_uuid: "19b1c7f7cd02496591ba26c8055f16e0" member_type: VOTER last_known_addr { host: "127.27.210.61" port: 44953 } } peers { permanent_uuid: "11fb2ea3421a4198a1f721b2f56be819" member_type: VOTER last_known_addr { host: "127.27.210.60" port: 45231 } }
06:35:41.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:41.892140 29073 leader_election.cc:288] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 912cbf41d7554146b0bd563d78e5ad17 (127.27.210.62:34959), 19b1c7f7cd02496591ba26c8055f16e0 (127.27.210.61:44953)
06:35:41.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:41.896368 28796 leader_election.cc:334] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 912cbf41d7554146b0bd563d78e5ad17 (127.27.210.62:34959): Network error: Client connection negotiation failed: client connection to 127.27.210.62:34959: connect: Connection refused (error 111)
06:35:41.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:41.896970 28798 leader_election.cc:334] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 19b1c7f7cd02496591ba26c8055f16e0 (127.27.210.61:44953): Network error: Client connection negotiation failed: client connection to 127.27.210.61:44953: connect: Connection refused (error 111)
06:35:41.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:41.897325 28798 leader_election.cc:302] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 11fb2ea3421a4198a1f721b2f56be819; no voters: 19b1c7f7cd02496591ba26c8055f16e0, 912cbf41d7554146b0bd563d78e5ad17
06:35:41.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:41.897969 29073 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:35:42.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:42.569832 28864 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 11fb2ea3421a4198a1f721b2f56be819: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:35:42.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:42.570063 28864 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:35:43.136 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.61:44953(127.27.210.61:44953)] connecting to peer
06:35:43.137 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.27.210.62:34959(127.27.210.62:34959)] connecting to peer
06:35:43.137 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:43.137 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.210.61:44953(127.27.210.61:44953)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:43.138 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.61:44953: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953
06:35:43.138 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:43.138 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.62:34959(127.27.210.62:34959)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:43.138 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.210.62:34959: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
06:35:43.140 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959; will retry
06:35:43.144 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29885), Traces: [0ms] refreshing cache from master, [66ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [152ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [228ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [307ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [647ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [651ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [656ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [662ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [663ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [664ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [666ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [668ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [679ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [681ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [682ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [683ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [701ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [702ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [702ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [705ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [740ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [741ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [742ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [743ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [782ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [782ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [783ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [784ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [796ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [801ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [802ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [803ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [804ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [936ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [940ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [941ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [943ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [945ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [1056ms] refreshing cache from master, [1056ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [1059ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [1059ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [1060ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [1063ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [1556ms] refreshing cache from master, [1556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [1557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [1558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [1560ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [1561ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [1562ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [1563ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [2556ms] refreshing cache from master, [2556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [2557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [2558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [2559ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [2561ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [2563ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [2564ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [4496ms] refreshing cache from master, [4496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [4497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [4498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [4500ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [4500ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [4501ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [4501ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [7756ms] refreshing cache from master, [7757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [7757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [7758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [7759ms] too many traces: truncated at 100 traces, deferred=Deferred@1092844197(state=PENDING, result=null, callback=(continuation of Deferred@1206362733 after retry RPC after error@1358134021) -> (continuation of Deferred@138240068 after retry RPC after error@1376896436) -> (continuation of Deferred@673177937 after retry RPC after error@1677696004) -> (continuation of Deferred@2359167 after retry RPC after error@1547937450) -> (continuation of Deferred@193057050 after retry RPC after error@344024018) -> (continuation of Deferred@189181498 after retry RPC after error@1248935694) -> (continuation of Deferred@1341992539 after retry RPC after error@1358777221) -> (continuation of Deferred@1853102302 after retry RPC after error@767480657) -> (continuation of Deferred@194490187 after retry RPC after error@822901983) -> (continuation of Deferred@2138058952 after retry RPC after error@613367852) -> (continuation of Deferred@307514881 after retry RPC after error@1080832528) -> (continuation of Deferred@1046789285 after retry RPC after error@1614021989) -> (continuation of Deferred@1344301860 after retry RPC after error@1912418178) -> (continuation of Deferred@1793537164 after retry RPC after error@708320152) -> (continuation of Deferred@2031234224 after retry RPC after error@1628151087) -> (continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226) -> (continuation of Deferred@1964987732 after retry RPC after error@1394346114) -> (continuation of Deferred@66619908 after retry RPC after error@1439002005) -> (continuation of Deferred@1526675340 after retry RPC after error@1762950217) -> (continuation of Deferred@788227013 after retry RPC after error@1353456202) -> (continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068), errback=(continuation of Deferred@1206362733 after retry RPC after error@1358134021) -> (continuation of Deferred@138240068 after retry RPC after error@1376896436) -> (continuation of Deferred@673177937 after retry RPC after error@1677696004) -> (continuation of Deferred@2359167 after retry RPC after error@1547937450) -> (continuation of Deferred@193057050 after retry RPC after error@344024018) -> (continuation of Deferred@189181498 after retry RPC after error@1248935694) -> (continuation of Deferred@1341992539 after retry RPC after error@1358777221) -> (continuation of Deferred@1853102302 after retry RPC after error@767480657) -> (continuation of Deferred@194490187 after retry RPC after error@822901983) -> (continuation of Deferred@2138058952 after retry RPC after error@613367852) -> (continuation of Deferred@307514881 after retry RPC after error@1080832528) -> (continuation of Deferred@1046789285 after retry RPC after error@1614021989) -> (continuation of Deferred@1344301860 after retry RPC after error@1912418178) -> (continuation of Deferred@1793537164 after retry RPC after error@708320152) -> (continuation of Deferred@2031234224 after retry RPC after error@1628151087) -> (continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226) -> (continuation of Deferred@1964987732 after retry RPC after error@1394346114) -> (continuation of Deferred@66619908 after retry RPC after error@1439002005) -> (continuation of Deferred@1526675340 after retry RPC after error@1762950217) -> (continuation of Deferred@788227013 after retry RPC after error@1353456202) -> (continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068)))
06:35:43.149 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.27.210.62:34959(127.27.210.62:34959)] disconnecting while in state TERMINATED
06:35:43.150 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.27.210.60:45231(127.27.210.60:45231)] disconnecting while in state READY
06:35:43.150 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.27.210.61:44953(127.27.210.61:44953)] disconnecting while in state TERMINATED
06:35:43.155 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
06:35:43.159 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
06:35:43.159 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.27.210.60:45231(127.27.210.60:45231)] handling channelInactive
06:35:43.159 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.210.60:45231(127.27.210.60:45231)] cleaning up while in state READY due to: connection closed
06:35:43.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:43.166124 28488 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu with pid 28777
06:35:43.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:43.256484 28333 debug-util.cc:398] Leaking SignalData structure 0x7b08000af0e0 after lost signal to thread 28207
06:35:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 06:35:43.257578 28333 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7e80 after lost signal to thread 28336
06:35:44.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T06:35:44Z chronyd exiting
06:35:45.420 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
06:35:45.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:45.421322 27762 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu with pid 28043
06:35:45.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:45.449426 27762 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu with pid 28202
06:35:45.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:45.477463 27762 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu with pid 28347
06:35:45.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:45.502637 27762 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu with pid 27774
06:35:45.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:45.536988 27762 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu with pid 27844
06:35:45.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 06:35:45.567344 27762 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskYC5Dsr/build/tsan/bin/kudu with pid 27919
06:35:46.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T06:35:46Z chronyd exiting
06:35:47.761 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29885), Traces: [0ms] refreshing cache from master, [66ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [152ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [228ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [307ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [647ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [651ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [656ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [662ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [663ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [664ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [666ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [668ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [679ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [681ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [682ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [683ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [701ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [702ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [702ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [705ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [740ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [741ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [742ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [743ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [782ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [782ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [783ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [784ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [796ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [801ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [802ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [803ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [804ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [936ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [940ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [941ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [943ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [945ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [1056ms] refreshing cache from master, [1056ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [1059ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [1059ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [1060ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [1063ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [1556ms] refreshing cache from master, [1556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [1557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [1558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [1560ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [1561ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [1562ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [1563ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [2556ms] refreshing cache from master, [2556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [2557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [2558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [2559ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [2561ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [2563ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [2564ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [4496ms] refreshing cache from master, [4496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [4497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [4498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [4500ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [4500ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [4501ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [4501ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [7756ms] refreshing cache from master, [7757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [7757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [7758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [7759ms] too many traces: truncated at 100 traces, deferred=Deferred@1092844197(state=PENDING, result=null, callback=(continuation of Deferred@1206362733 after retry RPC after error@1358134021) -> (continuation of Deferred@138240068 after retry RPC after error@1376896436) -> (continuation of Deferred@673177937 after retry RPC after error@1677696004) -> (continuation of Deferred@2359167 after retry RPC after error@1547937450) -> (continuation of Deferred@193057050 after retry RPC after error@344024018) -> (continuation of Deferred@189181498 after retry RPC after error@1248935694) -> (continuation of Deferred@1341992539 after retry RPC after error@1358777221) -> (continuation of Deferred@1853102302 after retry RPC after error@767480657) -> (continuation of Deferred@194490187 after retry RPC after error@822901983) -> (continuation of Deferred@2138058952 after retry RPC after error@613367852) -> (continuation of Deferred@307514881 after retry RPC after error@1080832528) -> (continuation of Deferred@1046789285 after retry RPC after error@1614021989) -> (continuation of Deferred@1344301860 after retry RPC after error@1912418178) -> (continuation of Deferred@1793537164 after retry RPC after error@708320152) -> (continuation of Deferred@2031234224 after retry RPC after error@1628151087) -> (continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226) -> (continuation of Deferred@1964987732 after retry RPC after error@1394346114) -> (continuation of Deferred@66619908 after retry RPC after error@1439002005) -> (continuation of Deferred@1526675340 after retry RPC after error@1762950217) -> (continuation of Deferred@788227013 after retry RPC after error@1353456202) -> (continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068), errback=(continuation of Deferred@1206362733 after retry RPC after error@1358134021) -> (continuation of Deferred@138240068 after retry RPC after error@1376896436) -> (continuation of Deferred@673177937 after retry RPC after error@1677696004) -> (continuation of Deferred@2359167 after retry RPC after error@1547937450) -> (continuation of Deferred@193057050 after retry RPC after error@344024018) -> (continuation of Deferred@189181498 after retry RPC after error@1248935694) -> (continuation of Deferred@1341992539 after retry RPC after error@1358777221) -> (continuation of Deferred@1853102302 after retry RPC after error@767480657) -> (continuation of Deferred@194490187 after retry RPC after error@822901983) -> (continuation of Deferred@2138058952 after retry RPC after error@613367852) -> (continuation of Deferred@307514881 after retry RPC after error@1080832528) -> (continuation of Deferred@1046789285 after retry RPC after error@1614021989) -> (continuation of Deferred@1344301860 after retry RPC after error@1912418178) -> (continuation of Deferred@1793537164 after retry RPC after error@708320152) -> (continuation of Deferred@2031234224 after retry RPC after error@1628151087) -> (continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226) -> (continuation of Deferred@1964987732 after retry RPC after error@1394346114) -> (continuation of Deferred@66619908 after retry RPC after error@1439002005) -> (continuation of Deferred@1526675340 after retry RPC after error@1762950217) -> (continuation of Deferred@788227013 after retry RPC after error@1353456202) -> (continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.27.210.62:34959
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29885), Traces: [0ms] refreshing cache from master, [66ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [152ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [228ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [307ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [647ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [651ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [656ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [662ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [663ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [664ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [666ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [668ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [679ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [681ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [682ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [683ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [701ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [702ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [702ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [705ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [736ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [740ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [741ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [742ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [743ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [782ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [782ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [783ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [784ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [796ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [801ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [802ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [803ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [804ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [936ms] refreshing cache from master, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [940ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [941ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [943ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [945ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [1056ms] refreshing cache from master, [1056ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [1059ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [1059ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [1060ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [1063ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [1556ms] refreshing cache from master, [1556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [1557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [1558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [1560ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [1561ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [1562ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [1563ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [2556ms] refreshing cache from master, [2556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [2557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [2558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [2559ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [2561ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [2563ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [2564ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [4496ms] refreshing cache from master, [4496ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [4497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [4498ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [4500ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.62:34959: Network error: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959, [4500ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.61:44953: Network error: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [4501ms] Sub RPC ConnectToMaster: received response from server master-127.27.210.60:45231: OK, [4501ms] delaying RPC due to: Service unavailable: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953, [7756ms] refreshing cache from master, [7757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.60:45231, [7757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.61:44953, [7758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.210.62:34959, [7759ms] too many traces: truncated at 100 traces, deferred=Deferred@1092844197(state=PENDING, result=null, callback=(continuation of Deferred@1206362733 after retry RPC after error@1358134021) -> (continuation of Deferred@138240068 after retry RPC after error@1376896436) -> (continuation of Deferred@673177937 after retry RPC after error@1677696004) -> (continuation of Deferred@2359167 after retry RPC after error@1547937450) -> (continuation of Deferred@193057050 after retry RPC after error@344024018) -> (continuation of Deferred@189181498 after retry RPC after error@1248935694) -> (continuation of Deferred@1341992539 after retry RPC after error@1358777221) -> (continuation of Deferred@1853102302 after retry RPC after error@767480657) -> (continuation of Deferred@194490187 after retry RPC after error@822901983) -> (continuation of Deferred@2138058952 after retry RPC after error@613367852) -> (continuation of Deferred@307514881 after retry RPC after error@1080832528) -> (continuation of Deferred@1046789285 after retry RPC after error@1614021989) -> (continuation of Deferred@1344301860 after retry RPC after error@1912418178) -> (continuation of Deferred@1793537164 after retry RPC after error@708320152) -> (continuation of Deferred@2031234224 after retry RPC after error@1628151087) -> (continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226) -> (continuation of Deferred@1964987732 after retry RPC after error@1394346114) -> (continuation of Deferred@66619908 after retry RPC after error@1439002005) -> (continuation of Deferred@1526675340 after retry RPC after error@1762950217) -> (continuation of Deferred@788227013 after retry RPC after error@1353456202) -> (continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068), errback=(continuation of Deferred@1206362733 after retry RPC after error@1358134021) -> (continuation of Deferred@138240068 after retry RPC after error@1376896436) -> (continuation of Deferred@673177937 after retry RPC after error@1677696004) -> (continuation of Deferred@2359167 after retry RPC after error@1547937450) -> (continuation of Deferred@193057050 after retry RPC after error@344024018) -> (continuation of Deferred@189181498 after retry RPC after error@1248935694) -> (continuation of Deferred@1341992539 after retry RPC after error@1358777221) -> (continuation of Deferred@1853102302 after retry RPC after error@767480657) -> (continuation of Deferred@194490187 after retry RPC after error@822901983) -> (continuation of Deferred@2138058952 after retry RPC after error@613367852) -> (continuation of Deferred@307514881 after retry RPC after error@1080832528) -> (continuation of Deferred@1046789285 after retry RPC after error@1614021989) -> (continuation of Deferred@1344301860 after retry RPC after error@1912418178) -> (continuation of Deferred@1793537164 after retry RPC after error@708320152) -> (continuation of Deferred@2031234224 after retry RPC after error@1628151087) -> (continuation of Deferred@849057494 after retry RPC after error@1486204870) -> (continuation of Deferred@805990344 after retry RPC after error@223702531) -> (continuation of Deferred@1670804569 after retry RPC after error@369626226) -> (continuation of Deferred@1964987732 after retry RPC after error@1394346114) -> (continuation of Deferred@66619908 after retry RPC after error@1439002005) -> (continuation of Deferred@1526675340 after retry RPC after error@1762950217) -> (continuation of Deferred@788227013 after retry RPC after error@1353456202) -> (continuation of Deferred@1934361732 after retry RPC after error@972085392) -> (continuation of Deferred@1281384736 after retry RPC after error@2083409068)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.27.210.60:45231,127.27.210.61:44953,127.27.210.62:34959) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.61:44953(127.27.210.61:44953): Connection refused: /127.27.210.61:44953,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.27.210.62:34959(127.27.210.62:34959): Connection refused: /127.27.210.62:34959
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.27.210.62:34959
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)