07:13:52.992 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
07:13:53.004 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskrvi3SN/test-tmp
07:13:53.007 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
07:13:53.010 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskrvi3SN/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
07:13:53.289 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
07:13:53.533 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:13:53.535 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
07:13:53.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:53.539409 8952 mini_oidc.cc:152] Starting JWKS server
07:13:53.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:53.544481 8952 webserver.cc:460] Webserver started at http://127.0.0.1:42613/ using document root <none> and password file <none>
07:13:53.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:53.545168 8952 mini_oidc.cc:189] Starting OIDC Discovery server
07:13:53.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:53.547443 8952 webserver.cc:460] Webserver started at http://127.0.0.1:41757/ using document root <none> and password file <none>
07:13:53.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T07:13:53Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:13:53.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T07:13:53Z Disabled control of system clock
07:13:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:53.603888 8952 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:13:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:13:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-0/wal
07:13:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-0/data
07:13:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:13:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:13:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:13:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:13:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:13:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:13:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-0/logs
07:13:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-0/data/info.pb
07:13:53.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:13:53.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:13:53.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:13:53.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:13:53.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:13:53.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:13:53.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:13:53.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:13:53.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:13:53.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:13:53.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:13:53.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:13:53.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.190.62:35347
07:13:53.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.190.62
07:13:53.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:13:53.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.190.20:44409
07:13:53.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:13:53.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:13:53.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:13:53.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:13:53.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.190.62:35347,127.8.190.61:45537,127.8.190.60:42833
07:13:53.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:13:53.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42613/jwks.json with env {}
07:13:53.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:53.919269 8967 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:13:53.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:53.919934 8967 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:13:53.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:53.920504 8967 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:13:53.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:53.951396 8967 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:13:53.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:53.951678 8967 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:13:53.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:53.951869 8967 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:13:53.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:53.952160 8967 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:13:53.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:53.952422 8967 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:13:53.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:53.952616 8967 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42613/jwks.json
07:13:53.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:53.989853 8967 master_runner.cc:386] Master server non-default flags:
07:13:53.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:13:53.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.190.20:44409
07:13:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:13:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:13:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-0/data
07:13:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-0/wal
07:13:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:13:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.190.62:35347,127.8.190.61:45537,127.8.190.60:42833
07:13:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:13:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:13:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:13:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.190.62:35347
07:13:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:13:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:13:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:13:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42613/jwks.json
07:13:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:13:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:13:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-0/data/info.pb
07:13:53.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.190.62
07:13:53.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:13:53.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:13:53.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:13:53.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:13:53.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:13:53.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:13:53.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-0/logs
07:13:53.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:13:53.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:13:53.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:13:53.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:13:53.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:13:53.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:13:53.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:13:53.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 06:34:24 UTC on ccycloud.quasar-rbacaa.root.comops.site
07:13:53.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575854
07:13:53.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:13:53.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:53.991711 8967 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:13:53.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:53.993530 8967 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:13:54.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:54.024772 8980 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:13:55.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:54.024822 8975 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:13:55.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:54.024829 8976 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:13:55.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:54.026676 8967 server_base.cc:1034] running on GCE node
07:13:55.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.290868 8967 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:13:55.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.293917 8967 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:13:55.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.295264 8967 hybrid_clock.cc:648] HybridClock initialized: now 1786259635295195 us; error 88 us; skew 500 ppm
07:13:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.296322 8967 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:13:55.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.307662 8967 webserver.cc:460] Webserver started at http://127.8.190.62:43823/ using document root <none> and password file <none>
07:13:55.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.308987 8967 fs_manager.cc:359] Metadata directory not provided
07:13:55.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.309182 8967 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:13:55.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.309685 8967 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:13:55.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.315332 8967 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-0/data/instance:
07:13:55.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75"
07:13:55.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:13:55 on dist-test-slave-1nl6"
07:13:55.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.316666 8967 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-0/wal/instance:
07:13:55.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75"
07:13:55.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:13:55 on dist-test-slave-1nl6"
07:13:55.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.325512 8967 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
07:13:55.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.332445 8993 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:13:55.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.333804 8967 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.001s sys 0.004s
07:13:55.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.334192 8967 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-0/data,/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-0/wal
07:13:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75"
07:13:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:13:55 on dist-test-slave-1nl6"
07:13:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.334563 8967 fs_report.cc:383] FS layout report
07:13:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:13:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-0/wal
07:13:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-0/wal
07:13:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-0/data/data
07:13:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:13:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:13:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:13:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:13:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:13:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:13:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:13:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:13:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:13:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:13:55.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:13:55.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.371963 8967 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:13:55.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.373622 8967 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:13:55.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.374083 8967 kserver.cc:163] Server-wide thread pool size limit: 3276
07:13:55.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.452557 8967 rpc_server.cc:307] RPC server started. Bound to: 127.8.190.62:35347
07:13:55.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.452638 9064 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.190.62:35347 every 8 connection(s)
07:13:55.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.455662 8967 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-0/data/info.pb
07:13:55.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.461024 8952 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu as pid 8967
07:13:55.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.461426 8952 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-0/wal/instance
07:13:55.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.464468 9066 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:13:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.465715 8952 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:13:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:13:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-1/wal
07:13:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-1/data
07:13:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:13:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:13:55.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:13:55.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:13:55.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:13:55.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:13:55.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-1/logs
07:13:55.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-1/data/info.pb
07:13:55.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:13:55.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:13:55.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:13:55.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:13:55.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:13:55.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:13:55.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:13:55.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:13:55.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:13:55.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:13:55.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:13:55.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:13:55.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.190.61:45537
07:13:55.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.190.61
07:13:55.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:13:55.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.190.20:44409
07:13:55.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:13:55.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:13:55.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:13:55.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:13:55.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.190.62:35347,127.8.190.61:45537,127.8.190.60:42833
07:13:55.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:13:55.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42613/jwks.json with env {}
07:13:55.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.479650 9066 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.190.62" port: 35347 } has no permanent_uuid. Determining permanent_uuid...
07:13:55.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.512646 9066 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } has no permanent_uuid. Determining permanent_uuid...
07:13:55.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:55.515111 8997 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.190.61:45537: connect: Connection refused (error 111)
07:13:55.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:55.519650 9066 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.190.61:45537: Network error: Client connection negotiation failed: client connection to 127.8.190.61:45537: connect: Connection refused (error 111)
07:13:55.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.569253 9066 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } attempt: 1
07:13:55.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:55.573216 9066 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.190.61:45537: Network error: Client connection negotiation failed: client connection to 127.8.190.61:45537: connect: Connection refused (error 111)
07:13:55.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.641700 9066 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } attempt: 2
07:13:55.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:55.645628 9066 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.190.61:45537: Network error: Client connection negotiation failed: client connection to 127.8.190.61:45537: connect: Connection refused (error 111)
07:13:55.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.737136 9066 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } attempt: 3
07:13:55.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:55.741410 9066 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.190.61:45537: Network error: Client connection negotiation failed: client connection to 127.8.190.61:45537: connect: Connection refused (error 111)
07:13:55.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:55.791211 9069 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:13:55.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:55.791785 9069 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:13:55.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:55.792351 9069 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:13:55.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:55.822919 9069 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:13:55.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:55.823196 9069 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:13:55.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:55.823396 9069 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:13:55.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:55.823573 9069 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:13:55.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:55.823750 9069 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:13:55.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:55.823974 9069 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42613/jwks.json
07:13:55.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.859422 9069 master_runner.cc:386] Master server non-default flags:
07:13:55.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:13:55.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.190.20:44409
07:13:55.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:13:55.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:13:55.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-1/data
07:13:55.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-1/wal
07:13:55.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:13:55.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.190.62:35347,127.8.190.61:45537,127.8.190.60:42833
07:13:55.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:13:55.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:13:55.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:13:55.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.190.61:45537
07:13:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:13:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:13:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:13:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42613/jwks.json
07:13:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:13:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:13:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-1/data/info.pb
07:13:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.190.61
07:13:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:13:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:13:55.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:13:55.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:13:55.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:13:55.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:13:55.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-1/logs
07:13:55.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:13:55.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:13:55.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:13:55.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:13:55.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:13:55.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:13:55.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:13:55.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 06:34:24 UTC on ccycloud.quasar-rbacaa.root.comops.site
07:13:55.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575854
07:13:55.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:13:55.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.860733 9069 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:13:55.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.862572 9069 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:13:55.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:55.880633 9085 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:13:55.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.884824 9066 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } attempt: 4
07:13:55.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:55.889027 9066 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.190.61:45537: Network error: Client connection negotiation failed: client connection to 127.8.190.61:45537: connect: Connection refused (error 111)
07:13:56.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:56.188457 9066 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } attempt: 5
07:13:56.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:56.193869 9066 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.190.61:45537: Network error: Client connection negotiation failed: client connection to 127.8.190.61:45537: connect: Connection refused (error 111)
07:13:56.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:56.741293 9066 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } attempt: 6
07:13:56.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:56.749444 9066 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.190.61:45537: Network error: Client connection negotiation failed: client connection to 127.8.190.61:45537: connect: Connection refused (error 111)
07:13:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:55.883251 9089 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:13:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:55.883611 9069 server_base.cc:1034] running on GCE node
07:13:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:55.880894 9087 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:13:57.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.092590 9069 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:13:57.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.095556 9069 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:13:57.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.096909 9069 hybrid_clock.cc:648] HybridClock initialized: now 1786259637096867 us; error 76 us; skew 500 ppm
07:13:57.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.098001 9069 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:13:57.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.104637 9069 webserver.cc:460] Webserver started at http://127.8.190.61:45187/ using document root <none> and password file <none>
07:13:57.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.105595 9069 fs_manager.cc:359] Metadata directory not provided
07:13:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.105783 9069 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:13:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.106184 9069 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:13:57.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.111119 9069 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-1/data/instance:
07:13:57.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5500bc693e354facbe7773c58ed5b9d2"
07:13:57.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:13:57 on dist-test-slave-1nl6"
07:13:57.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.112253 9069 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-1/wal/instance:
07:13:57.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5500bc693e354facbe7773c58ed5b9d2"
07:13:57.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:13:57 on dist-test-slave-1nl6"
07:13:57.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.119809 9069 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
07:13:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.125315 9103 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:13:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.126523 9069 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
07:13:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.126822 9069 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-1/data,/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-1/wal
07:13:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5500bc693e354facbe7773c58ed5b9d2"
07:13:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:13:57 on dist-test-slave-1nl6"
07:13:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.127197 9069 fs_report.cc:383] FS layout report
07:13:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:13:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-1/wal
07:13:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-1/wal
07:13:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-1/data/data
07:13:57.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:13:57.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:13:57.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:13:57.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:13:57.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:13:57.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:13:57.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:13:57.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:13:57.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:13:57.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:13:57.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:13:57.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.165189 9069 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:13:57.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.166625 9069 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:13:57.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.167124 9069 kserver.cc:163] Server-wide thread pool size limit: 3276
07:13:57.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.240319 9069 rpc_server.cc:307] RPC server started. Bound to: 127.8.190.61:45537
07:13:57.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.240429 9181 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.190.61:45537 every 8 connection(s)
07:13:57.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.242851 9069 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-1/data/info.pb
07:13:57.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.248100 9183 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:13:57.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.249954 8952 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu as pid 9069
07:13:57.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.250418 8952 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-1/wal/instance
07:13:57.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.253482 8952 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-2/wal
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-2/data
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-2/logs
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-2/data/info.pb
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.190.60:42833
07:13:57.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.190.60
07:13:57.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:13:57.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.190.20:44409
07:13:57.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:13:57.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:13:57.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:13:57.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:13:57.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.190.62:35347,127.8.190.61:45537,127.8.190.60:42833
07:13:57.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:13:57.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42613/jwks.json with env {}
07:13:57.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.264143 9183 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.190.62" port: 35347 } has no permanent_uuid. Determining permanent_uuid...
07:13:57.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.288286 9183 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } has no permanent_uuid. Determining permanent_uuid...
07:13:57.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.305399 9183 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } has no permanent_uuid. Determining permanent_uuid...
07:13:57.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:57.307895 9106 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.190.60:42833: connect: Connection refused (error 111)
07:13:57.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:57.312390 9183 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.190.60:42833: Network error: Client connection negotiation failed: client connection to 127.8.190.60:42833: connect: Connection refused (error 111)
07:13:57.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.361987 9183 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } attempt: 1
07:13:57.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:57.365983 9183 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.190.60:42833: Network error: Client connection negotiation failed: client connection to 127.8.190.60:42833: connect: Connection refused (error 111)
07:13:57.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.434504 9183 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } attempt: 2
07:13:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:57.438531 9183 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.190.60:42833: Network error: Client connection negotiation failed: client connection to 127.8.190.60:42833: connect: Connection refused (error 111)
07:13:57.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.530037 9183 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } attempt: 3
07:13:57.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:57.533838 9183 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.190.60:42833: Network error: Client connection negotiation failed: client connection to 127.8.190.60:42833: connect: Connection refused (error 111)
07:13:57.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:57.589963 9185 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:13:57.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:57.590528 9185 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:13:57.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:57.591250 9185 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:13:57.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:57.621805 9185 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:13:57.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:57.622076 9185 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:13:57.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:57.622282 9185 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:13:57.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:57.622537 9185 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:13:57.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:57.622824 9185 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:13:57.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:57.623045 9185 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42613/jwks.json
07:13:57.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.657416 9185 master_runner.cc:386] Master server non-default flags:
07:13:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:13:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.190.20:44409
07:13:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:13:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:13:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-2/data
07:13:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-2/wal
07:13:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:13:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.190.62:35347,127.8.190.61:45537,127.8.190.60:42833
07:13:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:13:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:13:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:13:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.190.60:42833
07:13:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:13:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:13:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:13:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42613/jwks.json
07:13:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:13:57.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:13:57.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-2/data/info.pb
07:13:57.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.190.60
07:13:57.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:13:57.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:13:57.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:13:57.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:13:57.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:13:57.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:13:57.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-2/logs
07:13:57.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:13:57.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:13:57.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:13:57.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:13:57.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:13:57.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:13:57.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:13:57.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 06:34:24 UTC on ccycloud.quasar-rbacaa.root.comops.site
07:13:57.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575854
07:13:57.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:13:57.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.658965 9185 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:13:57.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.660964 9185 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:13:57.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.677307 9183 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } attempt: 4
07:13:57.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:57.680502 9203 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:13:57.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:57.683385 9183 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.190.60:42833: Network error: Client connection negotiation failed: client connection to 127.8.190.60:42833: connect: Connection refused (error 111)
07:13:57.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.809886 9066 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } attempt: 7
07:13:57.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.844311 9066 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } has no permanent_uuid. Determining permanent_uuid...
07:13:57.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:57.862223 9066 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.190.60:42833: Network error: Client connection negotiation failed: client connection to 127.8.190.60:42833: connect: Connection refused (error 111)
07:13:57.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.920857 9066 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } attempt: 1
07:13:57.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:57.927011 9066 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.190.60:42833: Network error: Client connection negotiation failed: client connection to 127.8.190.60:42833: connect: Connection refused (error 111)
07:13:57.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:57.982834 9183 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } attempt: 5
07:13:57.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:57.989746 9183 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.190.60:42833: Network error: Client connection negotiation failed: client connection to 127.8.190.60:42833: connect: Connection refused (error 111)
07:13:58.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:58.008677 9066 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } attempt: 2
07:13:58.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:58.014709 9066 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.190.60:42833: Network error: Client connection negotiation failed: client connection to 127.8.190.60:42833: connect: Connection refused (error 111)
07:13:58.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:58.100359 9066 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } attempt: 3
07:13:58.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:58.106460 9066 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.190.60:42833: Network error: Client connection negotiation failed: client connection to 127.8.190.60:42833: connect: Connection refused (error 111)
07:13:58.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:58.247141 9066 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } attempt: 4
07:13:58.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:58.256140 9066 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.190.60:42833: Network error: Client connection negotiation failed: client connection to 127.8.190.60:42833: connect: Connection refused (error 111)
07:13:58.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:58.537283 9183 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } attempt: 6
07:13:58.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:58.539750 9066 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } attempt: 5
07:13:58.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:58.546350 9066 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.190.60:42833: Network error: Client connection negotiation failed: client connection to 127.8.190.60:42833: connect: Connection refused (error 111)
07:13:58.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:58.559492 9183 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.190.60:42833: Network error: Client connection negotiation failed: client connection to 127.8.190.60:42833: connect: Connection refused (error 111)
07:13:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:57.680546 9200 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:13:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:57.683519 9199 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:13:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:58.950115 9202 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1267 milliseconds
07:13:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:58.950256 9185 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:13:58.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:58.951735 9185 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:13:58.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:58.955282 9185 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:13:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:58.956775 9185 hybrid_clock.cc:648] HybridClock initialized: now 1786259638956738 us; error 58 us; skew 500 ppm
07:13:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:58.957628 9185 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:13:58.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:58.964756 9185 webserver.cc:460] Webserver started at http://127.8.190.60:42227/ using document root <none> and password file <none>
07:13:58.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:58.965703 9185 fs_manager.cc:359] Metadata directory not provided
07:13:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:58.965903 9185 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:13:58.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:58.966305 9185 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:13:58.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:58.971411 9185 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-2/data/instance:
07:13:58.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "789fbdcc016c4db5b8b626fd68ff358d"
07:13:58.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:13:58 on dist-test-slave-1nl6"
07:13:58.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:58.972488 9185 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-2/wal/instance:
07:13:58.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "789fbdcc016c4db5b8b626fd68ff358d"
07:13:58.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:13:58 on dist-test-slave-1nl6"
07:13:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:58.981451 9185 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
07:13:58.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:58.987891 9219 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:13:58.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:58.989073 9185 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
07:13:58.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:58.989367 9185 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-2/data,/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-2/wal
07:13:58.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "789fbdcc016c4db5b8b626fd68ff358d"
07:13:58.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:13:58 on dist-test-slave-1nl6"
07:13:58.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:58.989794 9185 fs_report.cc:383] FS layout report
07:13:58.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:13:58.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-2/wal
07:13:58.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-2/wal
07:13:58.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-2/data/data
07:13:58.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:13:58.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:13:58.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:13:58.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:13:58.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:13:58.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:13:58.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:13:58.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:13:58.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:13:58.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:13:58.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:13:59.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.041963 9185 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:13:59.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.043534 9185 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:13:59.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.043967 9185 kserver.cc:163] Server-wide thread pool size limit: 3276
07:13:59.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.098853 9066 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } attempt: 6
07:13:59.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:59.105873 9066 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.8.190.60:42833: Network error: Client connection negotiation failed: client connection to 127.8.190.60:42833: connect: Connection refused (error 111)
07:13:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.121515 9185 rpc_server.cc:307] RPC server started. Bound to: 127.8.190.60:42833
07:13:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.121665 9302 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.190.60:42833 every 8 connection(s)
07:13:59.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.125288 9185 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-2/data/info.pb
07:13:59.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.131322 9303 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:13:59.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.132491 8952 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu as pid 9185
07:13:59.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.132817 8952 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/master-2/wal/instance
07:13:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.144901 9303 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.190.62" port: 35347 } has no permanent_uuid. Determining permanent_uuid...
07:13:59.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.176738 9303 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } has no permanent_uuid. Determining permanent_uuid...
07:13:59.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.188547 9303 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } has no permanent_uuid. Determining permanent_uuid...
07:13:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.207213 9303 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d: Bootstrap starting.
07:13:59.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.216051 9303 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d: Neither blocks nor log segments found. Creating new log.
07:13:59.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.218833 9303 log.cc:824] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d: Log is configured to *not* fsync() on all Append() calls
07:13:59.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.224869 9303 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d: No bootstrap required, opened a new log
07:13:59.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.250763 9303 raft_consensus.cc:348] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75" member_type: VOTER last_known_addr { host: "127.8.190.62" port: 35347 } } peers { permanent_uuid: "5500bc693e354facbe7773c58ed5b9d2" member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } } peers { permanent_uuid: "789fbdcc016c4db5b8b626fd68ff358d" member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } }
07:13:59.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.251566 9303 raft_consensus.cc:374] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:13:59.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.251883 9303 raft_consensus.cc:729] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 789fbdcc016c4db5b8b626fd68ff358d, State: Initialized, Role: FOLLOWER
07:13:59.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.253293 9303 consensus_queue.cc:260] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75" member_type: VOTER last_known_addr { host: "127.8.190.62" port: 35347 } } peers { permanent_uuid: "5500bc693e354facbe7773c58ed5b9d2" member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } } peers { permanent_uuid: "789fbdcc016c4db5b8b626fd68ff358d" member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } }
07:13:59.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.256939 9314 sys_catalog.cc:455] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75" member_type: VOTER last_known_addr { host: "127.8.190.62" port: 35347 } } peers { permanent_uuid: "5500bc693e354facbe7773c58ed5b9d2" member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } } peers { permanent_uuid: "789fbdcc016c4db5b8b626fd68ff358d" member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } } }
07:13:59.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.257916 9314 sys_catalog.cc:458] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [sys.catalog]: This master's current role is: FOLLOWER
07:13:59.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.258436 9303 sys_catalog.cc:564] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [sys.catalog]: configured and running, proceeding with master startup.
07:13:59.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:59.288470 9329 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:13:59.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:59.288831 9329 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:13:59.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:59.299463 9228 tablet.cc:2369] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:13:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.281711 9185 master_runner.cc:186] Error getting master registration for 127.8.190.62:35347: OK, instance_id { permanent_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75" instance_seqno: 1786259635431491 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:13:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.309854 9185 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:13:59.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.437013 9314 raft_consensus.cc:482] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:13:59.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.437559 9314 raft_consensus.cc:504] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75" member_type: VOTER last_known_addr { host: "127.8.190.62" port: 35347 } } peers { permanent_uuid: "5500bc693e354facbe7773c58ed5b9d2" member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } } peers { permanent_uuid: "789fbdcc016c4db5b8b626fd68ff358d" member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } }
07:13:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.440248 9314 leader_election.cc:288] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ecad2f3c00fc460f8ffb69f8f6a5eb75 (127.8.190.62:35347), 5500bc693e354facbe7773c58ed5b9d2 (127.8.190.61:45537)
07:13:59.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.619995 9183 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } attempt: 7
07:13:59.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.641830 9183 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5500bc693e354facbe7773c58ed5b9d2: Bootstrap starting.
07:13:59.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.649284 9183 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5500bc693e354facbe7773c58ed5b9d2: Neither blocks nor log segments found. Creating new log.
07:13:59.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.651582 9183 log.cc:824] T 00000000000000000000000000000000 P 5500bc693e354facbe7773c58ed5b9d2: Log is configured to *not* fsync() on all Append() calls
07:13:59.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.657559 9183 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5500bc693e354facbe7773c58ed5b9d2: No bootstrap required, opened a new log
07:13:59.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.688460 9183 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5500bc693e354facbe7773c58ed5b9d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75" member_type: VOTER last_known_addr { host: "127.8.190.62" port: 35347 } } peers { permanent_uuid: "5500bc693e354facbe7773c58ed5b9d2" member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } } peers { permanent_uuid: "789fbdcc016c4db5b8b626fd68ff358d" member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } }
07:13:59.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.689301 9183 raft_consensus.cc:374] T 00000000000000000000000000000000 P 5500bc693e354facbe7773c58ed5b9d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:13:59.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.689564 9183 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5500bc693e354facbe7773c58ed5b9d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5500bc693e354facbe7773c58ed5b9d2, State: Initialized, Role: FOLLOWER
07:13:59.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.690436 9183 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5500bc693e354facbe7773c58ed5b9d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75" member_type: VOTER last_known_addr { host: "127.8.190.62" port: 35347 } } peers { permanent_uuid: "5500bc693e354facbe7773c58ed5b9d2" member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } } peers { permanent_uuid: "789fbdcc016c4db5b8b626fd68ff358d" member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } }
07:13:59.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.693523 9337 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5500bc693e354facbe7773c58ed5b9d2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75" member_type: VOTER last_known_addr { host: "127.8.190.62" port: 35347 } } peers { permanent_uuid: "5500bc693e354facbe7773c58ed5b9d2" member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } } peers { permanent_uuid: "789fbdcc016c4db5b8b626fd68ff358d" member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } } }
07:13:59.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.694504 9337 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5500bc693e354facbe7773c58ed5b9d2 [sys.catalog]: This master's current role is: FOLLOWER
07:13:59.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.695399 9183 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5500bc693e354facbe7773c58ed5b9d2 [sys.catalog]: configured and running, proceeding with master startup.
07:13:59.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.698608 9144 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "789fbdcc016c4db5b8b626fd68ff358d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5500bc693e354facbe7773c58ed5b9d2" is_pre_election: true
07:13:59.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.700042 9144 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5500bc693e354facbe7773c58ed5b9d2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 789fbdcc016c4db5b8b626fd68ff358d in term 0.
07:13:59.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.702210 9223 leader_election.cc:302] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5500bc693e354facbe7773c58ed5b9d2, 789fbdcc016c4db5b8b626fd68ff358d; no voters:
07:13:59.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.703456 9314 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [term 0 FOLLOWER]: Leader pre-election won for term 1
07:13:59.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.703851 9314 raft_consensus.cc:482] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:13:59.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.704175 9314 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [term 0 FOLLOWER]: Advancing to term 1
07:13:59.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.712308 9314 raft_consensus.cc:504] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75" member_type: VOTER last_known_addr { host: "127.8.190.62" port: 35347 } } peers { permanent_uuid: "5500bc693e354facbe7773c58ed5b9d2" member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } } peers { permanent_uuid: "789fbdcc016c4db5b8b626fd68ff358d" member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } }
07:13:59.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.723835 9314 leader_election.cc:288] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [CANDIDATE]: Term 1 election: Requested vote from peers ecad2f3c00fc460f8ffb69f8f6a5eb75 (127.8.190.62:35347), 5500bc693e354facbe7773c58ed5b9d2 (127.8.190.61:45537)
07:13:59.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.724905 9144 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "789fbdcc016c4db5b8b626fd68ff358d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5500bc693e354facbe7773c58ed5b9d2"
07:13:59.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.725613 9144 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5500bc693e354facbe7773c58ed5b9d2 [term 0 FOLLOWER]: Advancing to term 1
07:13:59.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.732967 9144 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5500bc693e354facbe7773c58ed5b9d2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 789fbdcc016c4db5b8b626fd68ff358d in term 1.
07:13:59.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.733970 9223 leader_election.cc:302] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 5500bc693e354facbe7773c58ed5b9d2, 789fbdcc016c4db5b8b626fd68ff358d; no voters:
07:13:59.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.734731 9314 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [term 1 FOLLOWER]: Leader election won for term 1
07:13:59.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.736929 9314 raft_consensus.cc:686] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [term 1 LEADER]: Becoming Leader. State: Replica: 789fbdcc016c4db5b8b626fd68ff358d, State: Running, Role: LEADER
07:13:59.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.737977 9314 consensus_queue.cc:237] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75" member_type: VOTER last_known_addr { host: "127.8.190.62" port: 35347 } } peers { permanent_uuid: "5500bc693e354facbe7773c58ed5b9d2" member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } } peers { permanent_uuid: "789fbdcc016c4db5b8b626fd68ff358d" member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } }
07:13:59.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:59.771970 9354 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5500bc693e354facbe7773c58ed5b9d2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:13:59.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:13:59.773555 9354 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:13:59.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.773588 9352 sys_catalog.cc:455] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 789fbdcc016c4db5b8b626fd68ff358d. Latest consensus state: current_term: 1 leader_uuid: "789fbdcc016c4db5b8b626fd68ff358d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75" member_type: VOTER last_known_addr { host: "127.8.190.62" port: 35347 } } peers { permanent_uuid: "5500bc693e354facbe7773c58ed5b9d2" member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } } peers { permanent_uuid: "789fbdcc016c4db5b8b626fd68ff358d" member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } } }
07:13:59.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.774358 9352 sys_catalog.cc:458] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [sys.catalog]: This master's current role is: LEADER
07:13:59.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.787684 9358 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
07:13:59.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.795571 9358 catalog_manager.cc:1484] Initializing Kudu cluster ID...
07:13:59.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.743993 9069 master_runner.cc:186] Error getting master registration for 127.8.190.62:35347: OK, instance_id { permanent_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75" instance_seqno: 1786259635431491 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:13:59.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.801889 9069 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:13:59.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.823825 9144 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 5500bc693e354facbe7773c58ed5b9d2 [term 1 FOLLOWER]: Refusing update from remote peer 789fbdcc016c4db5b8b626fd68ff358d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:13:59.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.825546 9352 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [LEADER]: Connected to new peer: Peer: permanent_uuid: "5500bc693e354facbe7773c58ed5b9d2" member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:13:59.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.854600 9337 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5500bc693e354facbe7773c58ed5b9d2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 789fbdcc016c4db5b8b626fd68ff358d. Latest consensus state: current_term: 1 leader_uuid: "789fbdcc016c4db5b8b626fd68ff358d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75" member_type: VOTER last_known_addr { host: "127.8.190.62" port: 35347 } } peers { permanent_uuid: "5500bc693e354facbe7773c58ed5b9d2" member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } } peers { permanent_uuid: "789fbdcc016c4db5b8b626fd68ff358d" member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } } }
07:13:59.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.855491 9337 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5500bc693e354facbe7773c58ed5b9d2 [sys.catalog]: This master's current role is: FOLLOWER
07:13:59.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.865309 9361 mvcc.cc:205] Tried to move back new op lower bound from 7316519484698361856 to 7316519484388941824. Current Snapshot: MvccSnapshot[applied={T|T < 7316519484698361856}]
07:13:59.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.866492 9352 sys_catalog.cc:455] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "789fbdcc016c4db5b8b626fd68ff358d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75" member_type: VOTER last_known_addr { host: "127.8.190.62" port: 35347 } } peers { permanent_uuid: "5500bc693e354facbe7773c58ed5b9d2" member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } } peers { permanent_uuid: "789fbdcc016c4db5b8b626fd68ff358d" member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } } }
07:13:59.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.867340 9352 sys_catalog.cc:458] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [sys.catalog]: This master's current role is: LEADER
07:13:59.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.881320 9358 catalog_manager.cc:1347] Generated new cluster ID: 756613ce3f04473fa8da01b7b3f5f37b
07:13:59.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.881633 9358 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
07:13:59.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.881404 9337 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5500bc693e354facbe7773c58ed5b9d2 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "789fbdcc016c4db5b8b626fd68ff358d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75" member_type: VOTER last_known_addr { host: "127.8.190.62" port: 35347 } } peers { permanent_uuid: "5500bc693e354facbe7773c58ed5b9d2" member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } } peers { permanent_uuid: "789fbdcc016c4db5b8b626fd68ff358d" member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } } }
07:13:59.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.882206 9337 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5500bc693e354facbe7773c58ed5b9d2 [sys.catalog]: This master's current role is: FOLLOWER
07:13:59.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.913498 9358 catalog_manager.cc:1370] Generated new certificate authority record
07:13:59.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.919061 9358 catalog_manager.cc:1504] Loading token signing keys...
07:13:59.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.952344 9358 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d: Generated new TSK 0
07:13:59.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.957154 9358 catalog_manager.cc:1514] Initializing in-progress tserver states...
07:13:59.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:13:59.984099 8952 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:13:59.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:13:59.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-0/wal
07:13:59.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-0/data
07:13:59.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:13:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:13:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:13:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:13:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:13:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:13:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-0/logs
07:13:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-0/data/info.pb
07:13:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:13:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:13:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:13:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:13:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:13:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:13:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:13:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:13:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:13:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:13:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.190.1:0
07:13:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.190.1
07:13:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.190.1
07:13:59.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:13:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.190.62:35347,127.8.190.61:45537,127.8.190.60:42833
07:13:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.190.20:44409
07:13:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:13:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:13:59.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:14:00.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.139287 9066 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } attempt: 7
07:14:00.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.202752 9066 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ecad2f3c00fc460f8ffb69f8f6a5eb75: Bootstrap starting.
07:14:00.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.222522 9066 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ecad2f3c00fc460f8ffb69f8f6a5eb75: Neither blocks nor log segments found. Creating new log.
07:14:00.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.225356 9066 log.cc:824] T 00000000000000000000000000000000 P ecad2f3c00fc460f8ffb69f8f6a5eb75: Log is configured to *not* fsync() on all Append() calls
07:14:00.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.254218 9066 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ecad2f3c00fc460f8ffb69f8f6a5eb75: No bootstrap required, opened a new log
07:14:00.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.343191 9066 raft_consensus.cc:348] T 00000000000000000000000000000000 P ecad2f3c00fc460f8ffb69f8f6a5eb75 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75" member_type: VOTER last_known_addr { host: "127.8.190.62" port: 35347 } } peers { permanent_uuid: "5500bc693e354facbe7773c58ed5b9d2" member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } } peers { permanent_uuid: "789fbdcc016c4db5b8b626fd68ff358d" member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } }
07:14:00.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.356436 9066 raft_consensus.cc:374] T 00000000000000000000000000000000 P ecad2f3c00fc460f8ffb69f8f6a5eb75 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:14:00.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.357169 9066 raft_consensus.cc:729] T 00000000000000000000000000000000 P ecad2f3c00fc460f8ffb69f8f6a5eb75 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ecad2f3c00fc460f8ffb69f8f6a5eb75, State: Initialized, Role: FOLLOWER
07:14:00.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.358564 9066 consensus_queue.cc:260] T 00000000000000000000000000000000 P ecad2f3c00fc460f8ffb69f8f6a5eb75 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75" member_type: VOTER last_known_addr { host: "127.8.190.62" port: 35347 } } peers { permanent_uuid: "5500bc693e354facbe7773c58ed5b9d2" member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } } peers { permanent_uuid: "789fbdcc016c4db5b8b626fd68ff358d" member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } }
07:14:00.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.377452 9036 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "789fbdcc016c4db5b8b626fd68ff358d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75" is_pre_election: true
07:14:00.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.378321 9036 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ecad2f3c00fc460f8ffb69f8f6a5eb75 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 789fbdcc016c4db5b8b626fd68ff358d in term 0.
07:14:00.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.379297 9034 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "789fbdcc016c4db5b8b626fd68ff358d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75"
07:14:00.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.379971 9034 raft_consensus.cc:3037] T 00000000000000000000000000000000 P ecad2f3c00fc460f8ffb69f8f6a5eb75 [term 0 FOLLOWER]: Advancing to term 1
07:14:00.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.369532 9371 sys_catalog.cc:455] T 00000000000000000000000000000000 P ecad2f3c00fc460f8ffb69f8f6a5eb75 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75" member_type: VOTER last_known_addr { host: "127.8.190.62" port: 35347 } } peers { permanent_uuid: "5500bc693e354facbe7773c58ed5b9d2" member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } } peers { permanent_uuid: "789fbdcc016c4db5b8b626fd68ff358d" member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } } }
07:14:00.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.383455 9371 sys_catalog.cc:458] T 00000000000000000000000000000000 P ecad2f3c00fc460f8ffb69f8f6a5eb75 [sys.catalog]: This master's current role is: FOLLOWER
07:14:00.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:00.384848 9001 tablet.cc:2369] T 00000000000000000000000000000000 P ecad2f3c00fc460f8ffb69f8f6a5eb75: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:14:00.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.388250 9034 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ecad2f3c00fc460f8ffb69f8f6a5eb75 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 789fbdcc016c4db5b8b626fd68ff358d in term 1.
07:14:00.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.389757 9031 raft_consensus.cc:1261] T 00000000000000000000000000000000 P ecad2f3c00fc460f8ffb69f8f6a5eb75 [term 1 FOLLOWER]: Refusing update from remote peer 789fbdcc016c4db5b8b626fd68ff358d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:14:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.396945 9314 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [LEADER]: Connected to new peer: Peer: permanent_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75" member_type: VOTER last_known_addr { host: "127.8.190.62" port: 35347 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:14:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.397357 9066 sys_catalog.cc:564] T 00000000000000000000000000000000 P ecad2f3c00fc460f8ffb69f8f6a5eb75 [sys.catalog]: configured and running, proceeding with master startup.
07:14:00.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.513567 9371 sys_catalog.cc:455] T 00000000000000000000000000000000 P ecad2f3c00fc460f8ffb69f8f6a5eb75 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 789fbdcc016c4db5b8b626fd68ff358d. Latest consensus state: current_term: 1 leader_uuid: "789fbdcc016c4db5b8b626fd68ff358d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75" member_type: VOTER last_known_addr { host: "127.8.190.62" port: 35347 } } peers { permanent_uuid: "5500bc693e354facbe7773c58ed5b9d2" member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } } peers { permanent_uuid: "789fbdcc016c4db5b8b626fd68ff358d" member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } } }
07:14:00.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.515188 9371 sys_catalog.cc:458] T 00000000000000000000000000000000 P ecad2f3c00fc460f8ffb69f8f6a5eb75 [sys.catalog]: This master's current role is: FOLLOWER
07:14:00.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.516896 9383 sys_catalog.cc:455] T 00000000000000000000000000000000 P ecad2f3c00fc460f8ffb69f8f6a5eb75 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "789fbdcc016c4db5b8b626fd68ff358d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75" member_type: VOTER last_known_addr { host: "127.8.190.62" port: 35347 } } peers { permanent_uuid: "5500bc693e354facbe7773c58ed5b9d2" member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } } peers { permanent_uuid: "789fbdcc016c4db5b8b626fd68ff358d" member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } } }
07:14:00.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.518126 9383 sys_catalog.cc:458] T 00000000000000000000000000000000 P ecad2f3c00fc460f8ffb69f8f6a5eb75 [sys.catalog]: This master's current role is: FOLLOWER
07:14:00.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.526434 9396 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
07:14:00.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:00.692533 9366 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:14:00.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:00.693579 9366 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:14:00.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:00.694237 9366 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:14:00.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:00.728871 9366 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:14:00.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:00.729866 9366 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.190.1
07:14:00.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.765689 9366 tablet_server_runner.cc:78] Tablet server non-default flags:
07:14:00.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:14:00.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.190.20:44409
07:14:00.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:14:00.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:14:00.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-0/data
07:14:00.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-0/wal
07:14:00.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:14:00.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:14:00.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.190.1:0
07:14:00.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:14:00.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:14:00.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:14:00.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-0/data/info.pb
07:14:00.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.190.1
07:14:00.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:14:00.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.190.62:35347,127.8.190.61:45537,127.8.190.60:42833
07:14:00.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:14:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:14:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:14:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:14:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:14:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.190.1
07:14:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-0/logs
07:14:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:14:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:14:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:14:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:14:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:14:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:14:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:14:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 06:34:24 UTC on ccycloud.quasar-rbacaa.root.comops.site
07:14:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575854
07:14:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:14:00.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.767423 9366 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:14:00.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.769459 9366 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:14:00.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.780165 9354 catalog_manager.cc:1259] Loaded cluster ID: 756613ce3f04473fa8da01b7b3f5f37b
07:14:00.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.780491 9354 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 5500bc693e354facbe7773c58ed5b9d2: loading cluster ID for follower catalog manager: success
07:14:00.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.791481 9354 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 5500bc693e354facbe7773c58ed5b9d2: acquiring CA information for follower catalog manager: success
07:14:00.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.801359 9354 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 5500bc693e354facbe7773c58ed5b9d2: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:14:00.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:00.814543 9403 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:00.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.960036 9411 sys_catalog.cc:455] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "789fbdcc016c4db5b8b626fd68ff358d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecad2f3c00fc460f8ffb69f8f6a5eb75" member_type: VOTER last_known_addr { host: "127.8.190.62" port: 35347 } } peers { permanent_uuid: "5500bc693e354facbe7773c58ed5b9d2" member_type: VOTER last_known_addr { host: "127.8.190.61" port: 45537 } } peers { permanent_uuid: "789fbdcc016c4db5b8b626fd68ff358d" member_type: VOTER last_known_addr { host: "127.8.190.60" port: 42833 } } }
07:14:00.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:00.967123 9411 sys_catalog.cc:458] T 00000000000000000000000000000000 P 789fbdcc016c4db5b8b626fd68ff358d [sys.catalog]: This master's current role is: LEADER
07:14:01.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:01.562798 9396 catalog_manager.cc:1259] Loaded cluster ID: 756613ce3f04473fa8da01b7b3f5f37b
07:14:01.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:01.563238 9396 catalog_manager.cc:1552] T 00000000000000000000000000000000 P ecad2f3c00fc460f8ffb69f8f6a5eb75: loading cluster ID for follower catalog manager: success
07:14:01.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:01.625363 9396 catalog_manager.cc:1574] T 00000000000000000000000000000000 P ecad2f3c00fc460f8ffb69f8f6a5eb75: acquiring CA information for follower catalog manager: success
07:14:01.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:01.671676 9396 catalog_manager.cc:1602] T 00000000000000000000000000000000 P ecad2f3c00fc460f8ffb69f8f6a5eb75: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:14:02.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:02.203502 9401 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 9366
07:14:02.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:02.884279 9366 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 2.065s user 0.531s sys 0.935s
07:14:02.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:02.884624 9366 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 2.065s user 0.531s sys 0.935s
07:14:02.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:00.821422 9404 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:02.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:02.893990 9417 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:02.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.894039 9366 server_base.cc:1034] running on GCE node
07:14:02.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.896093 9366 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:14:02.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.898783 9366 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:14:02.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.900329 9366 hybrid_clock.cc:648] HybridClock initialized: now 1786259642900286 us; error 45 us; skew 500 ppm
07:14:02.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.901363 9366 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:14:02.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.911906 9366 webserver.cc:460] Webserver started at http://127.8.190.1:33833/ using document root <none> and password file <none>
07:14:02.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.913094 9366 fs_manager.cc:359] Metadata directory not provided
07:14:02.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.913360 9366 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:14:02.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.913902 9366 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:14:02.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.920795 9366 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-0/data/instance:
07:14:02.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ffe81939bb0435f92642b289f6e3192"
07:14:02.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:14:02 on dist-test-slave-1nl6"
07:14:02.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.922226 9366 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-0/wal/instance:
07:14:02.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ffe81939bb0435f92642b289f6e3192"
07:14:02.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:14:02 on dist-test-slave-1nl6"
07:14:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.931559 9366 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.006s sys 0.002s
07:14:02.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.938836 9424 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:14:02.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.940155 9366 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
07:14:02.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.940510 9366 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-0/data,/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-0/wal
07:14:02.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ffe81939bb0435f92642b289f6e3192"
07:14:02.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:14:02 on dist-test-slave-1nl6"
07:14:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.940937 9366 fs_report.cc:383] FS layout report
07:14:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:14:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-0/wal
07:14:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-0/wal
07:14:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-0/data/data
07:14:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:14:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:14:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:14:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:14:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:14:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:14:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:14:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:14:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:14:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:14:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:14:02.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.980672 9366 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:14:02.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.982424 9366 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:14:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.983026 9366 kserver.cc:163] Server-wide thread pool size limit: 3276
07:14:02.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.986208 9366 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:14:02.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.991534 9366 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:14:02.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.991853 9366 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:14:02.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.992166 9366 ts_tablet_manager.cc:610] Registered 0 tablets
07:14:02.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:02.992367 9366 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:14:03.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:03.178057 9366 rpc_server.cc:307] RPC server started. Bound to: 127.8.190.1:42321
07:14:03.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:03.178174 9586 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.190.1:42321 every 8 connection(s)
07:14:03.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:03.180783 9366 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-0/data/info.pb
07:14:03.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:03.183851 8952 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu as pid 9366
07:14:03.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:03.184461 8952 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-0/wal/instance
07:14:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:03.201974 8952 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:14:03.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:14:03.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-1/wal
07:14:03.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-1/data
07:14:03.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:14:03.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:14:03.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:14:03.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:14:03.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:14:03.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:14:03.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-1/logs
07:14:03.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-1/data/info.pb
07:14:03.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:14:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:14:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:14:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:14:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:14:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:14:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:14:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:14:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:14:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:14:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.190.2:0
07:14:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.190.2
07:14:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.190.2
07:14:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:14:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.190.62:35347,127.8.190.61:45537,127.8.190.60:42833
07:14:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.190.20:44409
07:14:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:14:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:14:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:14:03.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:03.241708 9587 heartbeater.cc:347] Connected to a master server at 127.8.190.60:42833
07:14:03.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:03.242206 9587 heartbeater.cc:464] Registering TS with master...
07:14:03.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:03.243619 9587 heartbeater.cc:511] Master 127.8.190.60:42833 requested a full tablet report, sending...
07:14:03.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:03.247387 9248 ts_manager.cc:194] Registered new tserver with Master: 2ffe81939bb0435f92642b289f6e3192 (127.8.190.1:42321)
07:14:03.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:03.250716 9248 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.190.1:56831
07:14:03.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:03.260080 9588 heartbeater.cc:347] Connected to a master server at 127.8.190.61:45537
07:14:03.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:03.260444 9588 heartbeater.cc:464] Registering TS with master...
07:14:03.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:03.261140 9588 heartbeater.cc:511] Master 127.8.190.61:45537 requested a full tablet report, sending...
07:14:03.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:03.264154 9131 ts_manager.cc:194] Registered new tserver with Master: 2ffe81939bb0435f92642b289f6e3192 (127.8.190.1:42321)
07:14:03.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:03.266327 9593 heartbeater.cc:347] Connected to a master server at 127.8.190.62:35347
07:14:03.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:03.266681 9593 heartbeater.cc:464] Registering TS with master...
07:14:03.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:03.267408 9593 heartbeater.cc:511] Master 127.8.190.62:35347 requested a full tablet report, sending...
07:14:03.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:03.271004 9017 ts_manager.cc:194] Registered new tserver with Master: 2ffe81939bb0435f92642b289f6e3192 (127.8.190.1:42321)
07:14:03.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:03.722661 9601 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:14:03.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:03.723294 9601 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:14:03.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:03.723929 9601 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:14:03.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:03.757233 9601 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:14:03.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:03.758507 9601 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.190.2
07:14:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:03.795099 9601 tablet_server_runner.cc:78] Tablet server non-default flags:
07:14:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:14:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.190.20:44409
07:14:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:14:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:14:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-1/data
07:14:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-1/wal
07:14:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:14:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:14:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.190.2:0
07:14:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:14:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:14:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:14:03.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-1/data/info.pb
07:14:03.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.190.2
07:14:03.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:14:03.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.190.62:35347,127.8.190.61:45537,127.8.190.60:42833
07:14:03.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:14:03.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:14:03.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:14:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:14:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:14:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.190.2
07:14:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-1/logs
07:14:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:14:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:14:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:14:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:14:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:14:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:14:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:14:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 06:34:24 UTC on ccycloud.quasar-rbacaa.root.comops.site
07:14:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575854
07:14:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:14:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:03.796729 9601 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:14:03.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:03.798959 9601 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:14:03.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:03.821022 9613 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:04.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:04.256014 9587 heartbeater.cc:503] Master 127.8.190.60:42833 was elected leader, sending a full tablet report...
07:14:05.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:03.821789 9616 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:05.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:03.821215 9612 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:05.050001 9614 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
07:14:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.050153 9601 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:14:05.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.053736 9601 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:14:05.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.056226 9601 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:14:05.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.057616 9601 hybrid_clock.cc:648] HybridClock initialized: now 1786259645057584 us; error 42 us; skew 500 ppm
07:14:05.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.058521 9601 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:14:05.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.065724 9601 webserver.cc:460] Webserver started at http://127.8.190.2:46185/ using document root <none> and password file <none>
07:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.066807 9601 fs_manager.cc:359] Metadata directory not provided
07:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.067152 9601 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:14:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.067689 9601 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:14:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.072790 9601 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-1/data/instance:
07:14:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8d3be6243fcf4cc98583253d9f765e8d"
07:14:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:14:05 on dist-test-slave-1nl6"
07:14:05.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.074021 9601 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-1/wal/instance:
07:14:05.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8d3be6243fcf4cc98583253d9f765e8d"
07:14:05.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:14:05 on dist-test-slave-1nl6"
07:14:05.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.081478 9601 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
07:14:05.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.089190 9627 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:14:05.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.090596 9601 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.002s sys 0.003s
07:14:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.091004 9601 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-1/data,/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-1/wal
07:14:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8d3be6243fcf4cc98583253d9f765e8d"
07:14:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:14:05 on dist-test-slave-1nl6"
07:14:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.091383 9601 fs_report.cc:383] FS layout report
07:14:05.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:14:05.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-1/wal
07:14:05.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-1/wal
07:14:05.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-1/data/data
07:14:05.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:14:05.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:14:05.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:14:05.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:14:05.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:14:05.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:14:05.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:14:05.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:14:05.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:14:05.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:14:05.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:14:05.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.161221 9601 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:14:05.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.164584 9601 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:14:05.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.165500 9601 kserver.cc:163] Server-wide thread pool size limit: 3276
07:14:05.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.173803 9601 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:14:05.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.186195 9601 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:14:05.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.186625 9601 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:14:05.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.187106 9601 ts_tablet_manager.cc:610] Registered 0 tablets
07:14:05.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.187443 9601 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:14:05.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.593878 9601 rpc_server.cc:307] RPC server started. Bound to: 127.8.190.2:33341
07:14:05.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.594108 9793 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.190.2:33341 every 8 connection(s)
07:14:05.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.596951 9601 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-1/data/info.pb
07:14:05.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.602115 8952 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu as pid 9601
07:14:05.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.603658 8952 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-1/wal/instance
07:14:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.654810 8952 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:14:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:14:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-2/wal
07:14:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-2/data
07:14:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:14:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:14:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:14:05.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:14:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:14:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:14:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-2/logs
07:14:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-2/data/info.pb
07:14:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:14:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:14:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:14:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:14:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:14:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:14:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:14:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:14:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:14:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:14:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.190.3:0
07:14:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.190.3
07:14:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.190.3
07:14:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:14:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.190.62:35347,127.8.190.61:45537,127.8.190.60:42833
07:14:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.190.20:44409
07:14:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:14:05.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:14:05.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:14:05.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.682163 9794 heartbeater.cc:347] Connected to a master server at 127.8.190.60:42833
07:14:05.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.682709 9794 heartbeater.cc:464] Registering TS with master...
07:14:05.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.684652 9794 heartbeater.cc:511] Master 127.8.190.60:42833 requested a full tablet report, sending...
07:14:05.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.688171 9248 ts_manager.cc:194] Registered new tserver with Master: 8d3be6243fcf4cc98583253d9f765e8d (127.8.190.2:33341)
07:14:05.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.690578 9248 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.190.2:58155
07:14:05.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.701054 9795 heartbeater.cc:347] Connected to a master server at 127.8.190.61:45537
07:14:05.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.701493 9795 heartbeater.cc:464] Registering TS with master...
07:14:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.702620 9795 heartbeater.cc:511] Master 127.8.190.61:45537 requested a full tablet report, sending...
07:14:05.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.705709 9798 heartbeater.cc:347] Connected to a master server at 127.8.190.62:35347
07:14:05.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.706238 9798 heartbeater.cc:464] Registering TS with master...
07:14:05.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.706809 9131 ts_manager.cc:194] Registered new tserver with Master: 8d3be6243fcf4cc98583253d9f765e8d (127.8.190.2:33341)
07:14:05.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.707154 9798 heartbeater.cc:511] Master 127.8.190.62:35347 requested a full tablet report, sending...
07:14:05.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:05.711144 9017 ts_manager.cc:194] Registered new tserver with Master: 8d3be6243fcf4cc98583253d9f765e8d (127.8.190.2:33341)
07:14:06.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:06.089036 9805 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:14:06.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:06.089605 9805 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:14:06.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:06.090221 9805 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:14:06.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:06.125267 9805 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:14:06.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:06.126632 9805 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.190.3
07:14:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:06.176100 9805 tablet_server_runner.cc:78] Tablet server non-default flags:
07:14:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:14:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.190.20:44409
07:14:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:14:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:14:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-2/data
07:14:06.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-2/wal
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.190.3:0
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-2/data/info.pb
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.190.3
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.190.62:35347,127.8.190.61:45537,127.8.190.60:42833
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.190.3
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-2/logs
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 06:34:24 UTC on ccycloud.quasar-rbacaa.root.comops.site
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575854
07:14:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:14:06.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:06.177752 9805 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:14:06.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:06.180154 9805 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:14:06.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:06.203037 9818 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:06.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:06.697885 9794 heartbeater.cc:503] Master 127.8.190.60:42833 was elected leader, sending a full tablet report...
07:14:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:06.203416 9821 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:06.205554 9805 server_base.cc:1034] running on GCE node
07:14:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:06.203799 9816 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:07.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.514104 9805 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:14:07.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.518071 9805 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:14:07.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.519510 9805 hybrid_clock.cc:648] HybridClock initialized: now 1786259647519455 us; error 52 us; skew 500 ppm
07:14:07.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.520560 9805 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:14:07.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.532862 9805 webserver.cc:460] Webserver started at http://127.8.190.3:41419/ using document root <none> and password file <none>
07:14:07.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.534160 9805 fs_manager.cc:359] Metadata directory not provided
07:14:07.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.534365 9805 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:14:07.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.534960 9805 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:14:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.540489 9805 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-2/data/instance:
07:14:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f25ef5850be04348b44662fa2a39a01f"
07:14:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:14:07 on dist-test-slave-1nl6"
07:14:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.541607 9805 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-2/wal/instance:
07:14:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f25ef5850be04348b44662fa2a39a01f"
07:14:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:14:07 on dist-test-slave-1nl6"
07:14:07.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.551118 9805 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.006s sys 0.005s
07:14:07.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.560685 9834 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:14:07.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.561820 9805 fs_manager.cc:725] Time spent opening block manager: real 0.007s user 0.000s sys 0.009s
07:14:07.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.562376 9805 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-2/data,/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-2/wal
07:14:07.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f25ef5850be04348b44662fa2a39a01f"
07:14:07.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:14:07 on dist-test-slave-1nl6"
07:14:07.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.562819 9805 fs_report.cc:383] FS layout report
07:14:07.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:14:07.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-2/wal
07:14:07.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-2/wal
07:14:07.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-2/data/data
07:14:07.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:14:07.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:14:07.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:14:07.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:14:07.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:14:07.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:14:07.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:14:07.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:14:07.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:14:07.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:14:07.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:14:07.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.636401 9805 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:14:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.638967 9805 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:14:07.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.639757 9805 kserver.cc:163] Server-wide thread pool size limit: 3276
07:14:07.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.646292 9805 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:14:07.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.653676 9805 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:14:07.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.653909 9805 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:14:07.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.654299 9805 ts_tablet_manager.cc:610] Registered 0 tablets
07:14:07.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.654501 9805 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:14:07.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.925173 9805 rpc_server.cc:307] RPC server started. Bound to: 127.8.190.3:37323
07:14:07.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.925518 9985 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.8.190.3:37323 every 8 connection(s)
07:14:07.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.928404 9805 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-2/data/info.pb
07:14:07.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.930455 8952 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu as pid 9805
07:14:07.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:07.931263 8952 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster2514916041658588970/ts-2/wal/instance
07:14:08.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:08.029450 9986 heartbeater.cc:347] Connected to a master server at 127.8.190.60:42833
07:14:08.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:08.030179 9986 heartbeater.cc:464] Registering TS with master...
07:14:08.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:08.032505 9986 heartbeater.cc:511] Master 127.8.190.60:42833 requested a full tablet report, sending...
07:14:08.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:08.037030 9248 ts_manager.cc:194] Registered new tserver with Master: f25ef5850be04348b44662fa2a39a01f (127.8.190.3:37323)
07:14:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:08.040669 9248 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.8.190.3:38279
07:14:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:08.041424 9990 heartbeater.cc:347] Connected to a master server at 127.8.190.62:35347
07:14:08.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:08.041817 9990 heartbeater.cc:464] Registering TS with master...
07:14:08.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:08.043251 9990 heartbeater.cc:511] Master 127.8.190.62:35347 requested a full tablet report, sending...
07:14:08.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:08.045989 9988 heartbeater.cc:347] Connected to a master server at 127.8.190.61:45537
07:14:08.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:08.046576 9988 heartbeater.cc:464] Registering TS with master...
07:14:08.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:08.047953 9988 heartbeater.cc:511] Master 127.8.190.61:45537 requested a full tablet report, sending...
07:14:08.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:08.047940 9017 ts_manager.cc:194] Registered new tserver with Master: f25ef5850be04348b44662fa2a39a01f (127.8.190.3:37323)
07:14:08.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:08.052330 9131 ts_manager.cc:194] Registered new tserver with Master: f25ef5850be04348b44662fa2a39a01f (127.8.190.3:37323)
07:14:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:08.061126 8952 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
07:14:08.080 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:14:08.082 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
07:14:08.093 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.8.190.62"
port: 35347
}
bound_http_address {
host: "127.8.190.62"
port: 43823
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.8.190.61"
port: 45537
}
bound_http_address {
host: "127.8.190.61"
port: 45187
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.8.190.60"
port: 42833
}
bound_http_address {
host: "127.8.190.60"
port: 42227
}
}
}
07:14:08.099 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
07:14:08.102 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.8.190.1"
port: 42321
}
bound_http_address {
host: "127.8.190.1"
port: 33833
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.8.190.2"
port: 33341
}
bound_http_address {
host: "127.8.190.2"
port: 46185
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.8.190.3"
port: 37323
}
bound_http_address {
host: "127.8.190.3"
port: 41419
}
}
}
07:14:08.102 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
07:14:08.352 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
07:14:08.517 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskrvi3SN/test-tmp
07:14:08.518 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
07:14:08.518 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskrvi3SN/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
07:14:08.529 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
07:14:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:09.048743 9986 heartbeater.cc:503] Master 127.8.190.60:42833 was elected leader, sending a full tablet report...
07:14:09.143 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:14:09.143 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
07:14:09.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:09.148512 10001 mini_oidc.cc:152] Starting JWKS server
07:14:09.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:09.157354 10001 webserver.cc:460] Webserver started at http://127.0.0.1:33153/ using document root <none> and password file <none>
07:14:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:09.157835 10001 mini_oidc.cc:189] Starting OIDC Discovery server
07:14:09.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:09.161743 10001 webserver.cc:460] Webserver started at http://127.0.0.1:40481/ using document root <none> and password file <none>
07:14:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
07:14:09.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/principal' for realm 'KRBTEST.COM',
07:14:09.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
07:14:09.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:09 dist-test-slave-1nl6 krb5kdc[10016](info): setting up network...
07:14:09.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
07:14:09.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:09 dist-test-slave-1nl6 krb5kdc[10016](info): set up 2 sockets
07:14:09.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:09 dist-test-slave-1nl6 krb5kdc[10016](info): commencing operation
07:14:09.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
07:14:09.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:09.965087 9981 debug-util.cc:398] Leaking SignalData structure 0x7b08000af060 after lost signal to thread 9810
07:14:09.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:09.966435 9981 debug-util.cc:398] Leaking SignalData structure 0x7b08000aff40 after lost signal to thread 9985
07:14:10.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:10.544270 10001 mini_kdc.cc:121] Time spent starting KDC: real 1.349s user 0.006s sys 0.021s
07:14:10.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
07:14:10.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:14:10.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
07:14:10.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
07:14:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:14:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
07:14:10.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
07:14:10.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:14:10.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
07:14:10.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:14:10.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/test-user.keytab.
07:14:10.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/test-user.keytab.
07:14:10.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:10 dist-test-slave-1nl6 krb5kdc[10016](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786259650, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:14:10.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:14:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T07:14:10Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:14:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T07:14:10Z Disabled control of system clock
07:14:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.196.126@KRBTEST.COM; defaulting to no policy
07:14:10.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:14:10.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.196.126@KRBTEST.COM" created.
07:14:10.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:14:10.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.196.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kudu_127.9.196.126.keytab.
07:14:10.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.196.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kudu_127.9.196.126.keytab.
07:14:10.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:10.782222 10001 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:14:10.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:14:10.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/wal
07:14:10.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/data
07:14:10.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:14:10.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:14:10.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:14:10.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:14:10.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/logs
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/data/info.pb
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.196.126:37581
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.196.126
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kudu_127.9.196.126.keytab
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.196.126
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.196.84:33847
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:14:10.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:14:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659
07:14:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:14:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33153/jwks.json
07:14:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:14:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:11.199466 10039 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:14:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:11.200049 10039 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:14:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:11.200629 10039 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:14:11.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:11.234287 10039 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:14:11.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:11.234582 10039 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:14:11.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:11.234874 10039 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:14:11.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:11.235136 10039 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:14:11.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:11.235337 10039 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:14:11.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:11.235522 10039 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:14:11.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:11.235661 10039 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33153/jwks.json
07:14:11.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:11.292331 10039 master_runner.cc:386] Master server non-default flags:
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.196.84:33847
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/data
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/wal
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kudu_127.9.196.126.keytab
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.196.126
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.196.126:37581
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33153/jwks.json
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/data/info.pb
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.196.126
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/logs
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 06:34:24 UTC on ccycloud.quasar-rbacaa.root.comops.site
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575854
07:14:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:14:11.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:11.295095 10039 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:14:11.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:11.296530 10039 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:14:11.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:11.340796 10047 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:12.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:11.340886 10049 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:12.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:11.347326 10046 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:12.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:12.921497 10048 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1581 milliseconds
07:14:12.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:12.921703 10039 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:14:12.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:12.923334 10039 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:14:12.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:12.929113 10039 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:14:12.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:12.930629 10039 hybrid_clock.cc:648] HybridClock initialized: now 1786259652930578 us; error 69 us; skew 500 ppm
07:14:12.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:12.931588 10039 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:14:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:12 dist-test-slave-1nl6 krb5kdc[10016](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786259652, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.196.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:14:12.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:12.941838 10039 init.cc:377] Logged in from keytab as kudu/127.9.196.126@KRBTEST.COM (short username kudu)
07:14:12.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:12.956298 10039 webserver.cc:460] Webserver started at http://127.9.196.126:34735/ using document root <none> and password file <none>
07:14:12.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:12.957257 10039 fs_manager.cc:359] Metadata directory not provided
07:14:12.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:12.957445 10039 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:14:12.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:12.957854 10039 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:14:12.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:12.964015 10039 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/data/instance:
07:14:12.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "90f79cce0c894390839c29d841baf254"
07:14:12.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:14:12 on dist-test-slave-1nl6"
07:14:12.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:12.965289 10039 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/wal/instance:
07:14:12.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "90f79cce0c894390839c29d841baf254"
07:14:12.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:14:12 on dist-test-slave-1nl6"
07:14:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:12.978155 10039 fs_manager.cc:691] Time spent creating directory manager: real 0.012s user 0.009s sys 0.004s
07:14:12.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:12.988169 10064 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:14:12.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:12.989598 10039 fs_manager.cc:725] Time spent opening block manager: real 0.008s user 0.003s sys 0.006s
07:14:12.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:12.989976 10039 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/data,/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/wal
07:14:12.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "90f79cce0c894390839c29d841baf254"
07:14:12.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:14:12 on dist-test-slave-1nl6"
07:14:12.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:12.990311 10039 fs_report.cc:383] FS layout report
07:14:12.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:14:12.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/wal
07:14:12.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/wal
07:14:12.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/data/data
07:14:12.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:14:12.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:14:12.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:14:12.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:14:12.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:14:12.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:14:12.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:14:12.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:14:12.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:14:12.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:14:12.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:14:13.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:13.053089 10039 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:14:13.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:13.061444 10039 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:14:13.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:13.062192 10039 kserver.cc:163] Server-wide thread pool size limit: 3276
07:14:13.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:13.362026 10039 rpc_server.cc:307] RPC server started. Bound to: 127.9.196.126:37581
07:14:13.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:13.362361 10141 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.196.126:37581 every 8 connection(s)
07:14:13.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:13.366587 10039 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/data/info.pb
07:14:13.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:13.368047 10001 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu as pid 10039
07:14:13.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:13.368592 10001 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/wal/instance
07:14:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:13.384791 10142 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:14:13.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.196.125@KRBTEST.COM; defaulting to no policy
07:14:13.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:14:13.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.196.125@KRBTEST.COM" created.
07:14:13.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:13.408527 10142 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.196.126" port: 37581 } has no permanent_uuid. Determining permanent_uuid...
07:14:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:14:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.196.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kudu_127.9.196.125.keytab.
07:14:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.196.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kudu_127.9.196.125.keytab.
07:14:13.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:13.427771 10001 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:14:13.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:14:13.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/wal
07:14:13.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/data
07:14:13.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:14:13.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:14:13.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:14:13.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:14:13.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:14:13.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:14:13.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/logs
07:14:13.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/data/info.pb
07:14:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:14:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:14:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:14:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:14:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:14:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:14:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:14:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:14:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:14:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:14:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:14:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:14:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.196.125:44031
07:14:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.196.125
07:14:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:14:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kudu_127.9.196.125.keytab
07:14:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.196.125
07:14:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:14:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:14:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:14:13.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.196.84:33847
07:14:13.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:14:13.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:14:13.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:14:13.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:14:13.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659
07:14:13.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:14:13.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33153/jwks.json
07:14:13.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:14:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:13 dist-test-slave-1nl6 krb5kdc[10016](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786259652, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.196.126@KRBTEST.COM for kudu/127.9.196.126@KRBTEST.COM
07:14:13.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:13.483883 10142 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.196.125" port: 44031 } has no permanent_uuid. Determining permanent_uuid...
07:14:13.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:13.488176 10070 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.196.125:44031: connect: Connection refused (error 111)
07:14:13.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:13.497903 10142 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.196.125:44031: Network error: Client connection negotiation failed: client connection to 127.9.196.125:44031: connect: Connection refused (error 111)
07:14:13.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:13.547627 10142 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.196.125" port: 44031 } attempt: 1
07:14:13.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:13.551587 10142 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.196.125:44031: Network error: Client connection negotiation failed: client connection to 127.9.196.125:44031: connect: Connection refused (error 111)
07:14:13.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:13.619966 10142 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.196.125" port: 44031 } attempt: 2
07:14:13.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:13.624939 10142 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.196.125:44031: Network error: Client connection negotiation failed: client connection to 127.9.196.125:44031: connect: Connection refused (error 111)
07:14:13.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:13.716495 10142 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.196.125" port: 44031 } attempt: 3
07:14:13.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:13.723142 10142 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.196.125:44031: Network error: Client connection negotiation failed: client connection to 127.9.196.125:44031: connect: Connection refused (error 111)
07:14:13.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:13.866770 10142 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.196.125" port: 44031 } attempt: 4
07:14:13.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:13.873442 10142 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.196.125:44031: Network error: Client connection negotiation failed: client connection to 127.9.196.125:44031: connect: Connection refused (error 111)
07:14:13.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:13.918558 10150 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:14:13.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:13.919420 10150 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:14:13.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:13.920141 10150 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:14:13.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:13.957455 10150 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:14:13.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:13.957686 10150 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:14:13.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:13.957950 10150 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:14:13.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:13.958146 10150 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:14:13.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:13.958341 10150 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:14:13.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:13.958529 10150 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:14:13.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:13.958665 10150 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33153/jwks.json
07:14:13.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:13.999353 10150 master_runner.cc:386] Master server non-default flags:
07:14:13.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:14:13.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.196.84:33847
07:14:13.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:14:13.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:14:13.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/data
07:14:13.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/wal
07:14:13.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kudu_127.9.196.125.keytab
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.196.125
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.196.125:44031
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33153/jwks.json
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/data/info.pb
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.196.125
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:14:14.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:14:14.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/logs
07:14:14.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:14:14.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:14:14.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:14:14.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:14:14.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:14:14.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:14:14.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:14:14.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 06:34:24 UTC on ccycloud.quasar-rbacaa.root.comops.site
07:14:14.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575854
07:14:14.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:14:14.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:14.000489 10150 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:14:14.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:14.001567 10150 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:14:14.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:14.038859 10165 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:14.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:14.172878 10142 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.196.125" port: 44031 } attempt: 5
07:14:14.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:14.177765 10142 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.196.125:44031: Network error: Client connection negotiation failed: client connection to 127.9.196.125:44031: connect: Connection refused (error 111)
07:14:14.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:14.725183 10142 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.196.125" port: 44031 } attempt: 6
07:14:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:14.733866 10142 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.196.125:44031: Network error: Client connection negotiation failed: client connection to 127.9.196.125:44031: connect: Connection refused (error 111)
07:14:15.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:14.047104 10150 server_base.cc:1034] running on GCE node
07:14:15.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:14.050014 10169 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:15.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:14.047111 10167 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:15.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.449009 10150 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:14:15.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.452181 10150 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:14:15.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.453574 10150 hybrid_clock.cc:648] HybridClock initialized: now 1786259655453537 us; error 49 us; skew 500 ppm
07:14:15.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.454511 10150 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:14:15.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:15 dist-test-slave-1nl6 krb5kdc[10016](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786259655, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.196.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:14:15.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.462510 10150 init.cc:377] Logged in from keytab as kudu/127.9.196.125@KRBTEST.COM (short username kudu)
07:14:15.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.473266 10150 webserver.cc:460] Webserver started at http://127.9.196.125:33179/ using document root <none> and password file <none>
07:14:15.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.474184 10150 fs_manager.cc:359] Metadata directory not provided
07:14:15.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.474383 10150 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:14:15.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.474784 10150 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:14:15.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.480943 10150 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/data/instance:
07:14:15.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d9b55128cc645dda23591f839d83028"
07:14:15.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:14:15 on dist-test-slave-1nl6"
07:14:15.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.482144 10150 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/wal/instance:
07:14:15.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d9b55128cc645dda23591f839d83028"
07:14:15.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:14:15 on dist-test-slave-1nl6"
07:14:15.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.492614 10150 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.008s sys 0.004s
07:14:15.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.502754 10190 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:14:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.504051 10150 fs_manager.cc:725] Time spent opening block manager: real 0.007s user 0.002s sys 0.005s
07:14:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.504420 10150 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/data,/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/wal
07:14:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d9b55128cc645dda23591f839d83028"
07:14:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:14:15 on dist-test-slave-1nl6"
07:14:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.504714 10150 fs_report.cc:383] FS layout report
07:14:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:14:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/wal
07:14:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/wal
07:14:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/data/data
07:14:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:14:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:14:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:14:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:14:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:14:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:14:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:14:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:14:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:14:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:14:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:14:15.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.562244 10150 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:14:15.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.569763 10150 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:14:15.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.570309 10150 kserver.cc:163] Server-wide thread pool size limit: 3276
07:14:15.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.794377 10142 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.196.125" port: 44031 } attempt: 7
07:14:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.810230 10150 rpc_server.cc:307] RPC server started. Bound to: 127.9.196.125:44031
07:14:15.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.810480 10266 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.196.125:44031 every 8 connection(s)
07:14:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.814855 10150 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/data/info.pb
07:14:15.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.826073 10001 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu as pid 10150
07:14:15.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.826570 10001 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/wal/instance
07:14:15.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.838171 10268 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:14:15.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.196.124@KRBTEST.COM; defaulting to no policy
07:14:15.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.858038 10268 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.196.126" port: 37581 } has no permanent_uuid. Determining permanent_uuid...
07:14:15.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:14:15.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.196.124@KRBTEST.COM" created.
07:14:15.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:15 dist-test-slave-1nl6 krb5kdc[10016](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786259652, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.196.126@KRBTEST.COM for kudu/127.9.196.125@KRBTEST.COM
07:14:15.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:15 dist-test-slave-1nl6 krb5kdc[10016](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786259655, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.196.125@KRBTEST.COM for kudu/127.9.196.126@KRBTEST.COM
07:14:15.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.941572 10268 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.196.125" port: 44031 } has no permanent_uuid. Determining permanent_uuid...
07:14:15.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:15.941938 10142 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } has no permanent_uuid. Determining permanent_uuid...
07:14:15.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:15.950347 10142 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.196.124:40659: Network error: Client connection negotiation failed: client connection to 127.9.196.124:40659: connect: Connection refused (error 111)
07:14:16.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:16.008860 10142 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } attempt: 1
07:14:16.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:16.014628 10142 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.196.124:40659: Network error: Client connection negotiation failed: client connection to 127.9.196.124:40659: connect: Connection refused (error 111)
07:14:16.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:16 dist-test-slave-1nl6 krb5kdc[10016](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786259655, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.196.125@KRBTEST.COM for kudu/127.9.196.125@KRBTEST.COM
07:14:16.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:16.034596 10268 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } has no permanent_uuid. Determining permanent_uuid...
07:14:16.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:16.037496 10194 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.196.124:40659: connect: Connection refused (error 111)
07:14:16.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:16.043870 10268 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.196.124:40659: Network error: Client connection negotiation failed: client connection to 127.9.196.124:40659: connect: Connection refused (error 111)
07:14:16.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:16.093457 10268 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } attempt: 1
07:14:16.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:16.096197 10142 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } attempt: 2
07:14:16.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:16.097473 10268 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.196.124:40659: Network error: Client connection negotiation failed: client connection to 127.9.196.124:40659: connect: Connection refused (error 111)
07:14:16.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:16.099946 10142 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.196.124:40659: Network error: Client connection negotiation failed: client connection to 127.9.196.124:40659: connect: Connection refused (error 111)
07:14:16.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:16.165951 10268 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } attempt: 2
07:14:16.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:16.169687 10268 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.196.124:40659: Network error: Client connection negotiation failed: client connection to 127.9.196.124:40659: connect: Connection refused (error 111)
07:14:16.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:16.185278 10142 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } attempt: 3
07:14:16.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:16.194016 10142 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.196.124:40659: Network error: Client connection negotiation failed: client connection to 127.9.196.124:40659: connect: Connection refused (error 111)
07:14:16.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:16.261094 10268 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } attempt: 3
07:14:16.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:16.264809 10268 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.196.124:40659: Network error: Client connection negotiation failed: client connection to 127.9.196.124:40659: connect: Connection refused (error 111)
07:14:16.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:16.334542 10142 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } attempt: 4
07:14:16.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:16.338068 10142 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.196.124:40659: Network error: Client connection negotiation failed: client connection to 127.9.196.124:40659: connect: Connection refused (error 111)
07:14:16.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:16.408373 10268 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } attempt: 4
07:14:16.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:16.412076 10268 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.196.124:40659: Network error: Client connection negotiation failed: client connection to 127.9.196.124:40659: connect: Connection refused (error 111)
07:14:16.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:14:16.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.196.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kudu_127.9.196.124.keytab.
07:14:16.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.196.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kudu_127.9.196.124.keytab.
07:14:16.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:16.475906 10001 mini_kdc.cc:262] Time spent creating service keytab for kudu/127.9.196.124: real 0.647s user 0.001s sys 0.015s
07:14:16.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:16.478385 10001 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/wal
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/data
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/logs
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/data/info.pb
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.196.124:40659
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.196.124
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kudu_127.9.196.124.keytab
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.196.124
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.196.84:33847
07:14:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:14:16.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:14:16.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:14:16.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:14:16.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659
07:14:16.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:14:16.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33153/jwks.json
07:14:16.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:14:16.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:16.621433 10142 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } attempt: 5
07:14:16.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:16.626240 10142 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.196.124:40659: Network error: Client connection negotiation failed: client connection to 127.9.196.124:40659: connect: Connection refused (error 111)
07:14:16.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:16.711468 10268 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } attempt: 5
07:14:16.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:16.716554 10268 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.196.124:40659: Network error: Client connection negotiation failed: client connection to 127.9.196.124:40659: connect: Connection refused (error 111)
07:14:16.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:16.961685 10281 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:14:16.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:16.962438 10281 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:14:16.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:16.963246 10281 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:14:16.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:16.998112 10281 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:14:16.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:16.998688 10281 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:14:16.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:16.999065 10281 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:14:16.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:16.999287 10281 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:14:16.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:16.999478 10281 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:14:16.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:16.999790 10281 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:14:17.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:17.000033 10281 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33153/jwks.json
07:14:17.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:17.040241 10281 master_runner.cc:386] Master server non-default flags:
07:14:17.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:14:17.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.196.84:33847
07:14:17.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:14:17.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:14:17.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/data
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/wal
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kudu_127.9.196.124.keytab
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.196.124
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.196.124:40659
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33153/jwks.json
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/data/info.pb
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.196.124
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:14:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:14:17.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/logs
07:14:17.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:14:17.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:14:17.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:14:17.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:14:17.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:14:17.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:14:17.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:14:17.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 06:34:24 UTC on ccycloud.quasar-rbacaa.root.comops.site
07:14:17.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575854
07:14:17.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:14:17.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:17.041844 10281 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:14:17.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:17.043186 10281 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:14:17.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:17.082135 10288 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:17.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:17.178754 10142 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } attempt: 6
07:14:17.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:17.209956 10142 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.196.124:40659: Network error: Client connection negotiation failed: client connection to 127.9.196.124:40659: connect: Connection refused (error 111)
07:14:17.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:17.263976 10268 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } attempt: 6
07:14:17.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:17.294232 10268 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.196.124:40659: Network error: Client connection negotiation failed: client connection to 127.9.196.124:40659: connect: Connection refused (error 111)
07:14:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:18.243642 10142 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } attempt: 7
07:14:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:18.286828 10142 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.196.124:40659: Network error: Client connection negotiation failed: client connection to 127.9.196.124:40659: connect: Connection refused (error 111)
07:14:18.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:18.354868 10268 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } attempt: 7
07:14:18.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:18.381691 10268 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.9.196.124:40659: Network error: Client connection negotiation failed: client connection to 127.9.196.124:40659: connect: Connection refused (error 111)
07:14:18.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:17.084864 10287 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:18.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:17.083657 10291 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:18.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:18.774495 10290 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1688 milliseconds
07:14:18.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:18.774631 10281 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:14:18.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:18.776232 10281 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:14:18.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:18.780840 10281 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:14:18.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:18.782260 10281 hybrid_clock.cc:648] HybridClock initialized: now 1786259658782228 us; error 37 us; skew 500 ppm
07:14:18.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:18.783149 10281 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:14:18.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:18 dist-test-slave-1nl6 krb5kdc[10016](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786259658, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.196.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:14:18.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:18.791615 10281 init.cc:377] Logged in from keytab as kudu/127.9.196.124@KRBTEST.COM (short username kudu)
07:14:18.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:18.802089 10281 webserver.cc:460] Webserver started at http://127.9.196.124:38743/ using document root <none> and password file <none>
07:14:18.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:18.803129 10281 fs_manager.cc:359] Metadata directory not provided
07:14:18.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:18.803330 10281 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:14:18.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:18.803834 10281 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:14:18.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:18.810007 10281 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/data/instance:
07:14:18.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aafc297b81ee4c9ca11d91a1200f3093"
07:14:18.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:14:18 on dist-test-slave-1nl6"
07:14:18.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:18.811327 10281 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/wal/instance:
07:14:18.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aafc297b81ee4c9ca11d91a1200f3093"
07:14:18.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:14:18 on dist-test-slave-1nl6"
07:14:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:18.824816 10281 fs_manager.cc:691] Time spent creating directory manager: real 0.013s user 0.008s sys 0.004s
07:14:18.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:18.833748 10316 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:14:18.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:18.834816 10281 fs_manager.cc:725] Time spent opening block manager: real 0.007s user 0.002s sys 0.005s
07:14:18.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:18.835284 10281 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/data,/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/wal
07:14:18.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aafc297b81ee4c9ca11d91a1200f3093"
07:14:18.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:14:18 on dist-test-slave-1nl6"
07:14:18.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:18.835698 10281 fs_report.cc:383] FS layout report
07:14:18.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:14:18.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/wal
07:14:18.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/wal
07:14:18.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/data/data
07:14:18.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:14:18.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:14:18.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:14:18.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:14:18.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:14:18.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:14:18.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:14:18.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:14:18.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:14:18.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:14:18.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:14:18.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:18.887496 10281 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:14:18.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:18.894428 10281 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:14:18.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:18.895241 10281 kserver.cc:163] Server-wide thread pool size limit: 3276
07:14:19.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.077175 10281 rpc_server.cc:307] RPC server started. Bound to: 127.9.196.124:40659
07:14:19.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.077260 10398 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.196.124:40659 every 8 connection(s)
07:14:19.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.080098 10281 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/data/info.pb
07:14:19.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.085868 10399 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:14:19.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.086542 10001 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu as pid 10281
07:14:19.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.087165 10001 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/wal/instance
07:14:19.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.108271 10399 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.196.126" port: 37581 } has no permanent_uuid. Determining permanent_uuid...
07:14:19.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:19 dist-test-slave-1nl6 krb5kdc[10016](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786259658, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.196.124@KRBTEST.COM for kudu/127.9.196.126@KRBTEST.COM
07:14:19.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:19 dist-test-slave-1nl6 krb5kdc[10016](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786259650, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.196.124@KRBTEST.COM
07:14:19.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.183714 10399 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.196.125" port: 44031 } has no permanent_uuid. Determining permanent_uuid...
07:14:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:19 dist-test-slave-1nl6 krb5kdc[10016](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786259658, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.196.124@KRBTEST.COM for kudu/127.9.196.125@KRBTEST.COM
07:14:19.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:19 dist-test-slave-1nl6 krb5kdc[10016](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786259650, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.196.125@KRBTEST.COM
07:14:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.227156 10399 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } has no permanent_uuid. Determining permanent_uuid...
07:14:19.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:19 dist-test-slave-1nl6 krb5kdc[10016](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786259658, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.196.124@KRBTEST.COM for kudu/127.9.196.124@KRBTEST.COM
07:14:19.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:19 dist-test-slave-1nl6 krb5kdc[10016](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786259650, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.196.126@KRBTEST.COM
07:14:19.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.275221 10399 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: Bootstrap starting.
07:14:19.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.279280 10001 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
07:14:19.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.292776 10399 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: Neither blocks nor log segments found. Creating new log.
07:14:19.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.298125 10399 log.cc:824] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: Log is configured to *not* fsync() on all Append() calls
07:14:19.310 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:14:19.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.309960 10399 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: No bootstrap required, opened a new log
07:14:19.311 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
07:14:19.322 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
07:14:19.323 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
07:14:19.326 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.9.196.126"
port: 37581
}
bound_http_address {
host: "127.9.196.126"
port: 34735
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.9.196.125"
port: 44031
}
bound_http_address {
host: "127.9.196.125"
port: 33179
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.9.196.124"
port: 40659
}
bound_http_address {
host: "127.9.196.124"
port: 38743
}
}
}
07:14:19.327 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
07:14:19.327 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:14:19.327 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
07:14:19.329 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
07:14:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.363338 10399 raft_consensus.cc:348] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "90f79cce0c894390839c29d841baf254" member_type: VOTER last_known_addr { host: "127.9.196.126" port: 37581 } } peers { permanent_uuid: "5d9b55128cc645dda23591f839d83028" member_type: VOTER last_known_addr { host: "127.9.196.125" port: 44031 } } peers { permanent_uuid: "aafc297b81ee4c9ca11d91a1200f3093" member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } }
07:14:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.364945 10399 raft_consensus.cc:374] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:14:19.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.365754 10399 raft_consensus.cc:729] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: aafc297b81ee4c9ca11d91a1200f3093, State: Initialized, Role: FOLLOWER
07:14:19.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.367687 10399 consensus_queue.cc:260] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "90f79cce0c894390839c29d841baf254" member_type: VOTER last_known_addr { host: "127.9.196.126" port: 37581 } } peers { permanent_uuid: "5d9b55128cc645dda23591f839d83028" member_type: VOTER last_known_addr { host: "127.9.196.125" port: 44031 } } peers { permanent_uuid: "aafc297b81ee4c9ca11d91a1200f3093" member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } }
07:14:19.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:19 dist-test-slave-1nl6 krb5kdc[10016](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786259659, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:14:19.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:14:19.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.376251 10415 sys_catalog.cc:455] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "90f79cce0c894390839c29d841baf254" member_type: VOTER last_known_addr { host: "127.9.196.126" port: 37581 } } peers { permanent_uuid: "5d9b55128cc645dda23591f839d83028" member_type: VOTER last_known_addr { host: "127.9.196.125" port: 44031 } } peers { permanent_uuid: "aafc297b81ee4c9ca11d91a1200f3093" member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } } }
07:14:19.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.377951 10415 sys_catalog.cc:458] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [sys.catalog]: This master's current role is: FOLLOWER
07:14:19.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.379423 10399 sys_catalog.cc:564] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [sys.catalog]: configured and running, proceeding with master startup.
07:14:19.383 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:14:19.386 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/krb5cc
07:14:19.397 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:14:19.399 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/krb5cc
07:14:19.400 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:14:19.408 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
07:14:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:19.407817 10325 tablet.cc:2369] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:14:19.411 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
07:14:19.411 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
07:14:19.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:19 dist-test-slave-1nl6 krb5kdc[10016](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786259659, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:14:19.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:14:19.470 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:14:19.471 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/krb5cc
07:14:19.472 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:14:19.473 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.196.126:37581
07:14:19.476 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
07:14:19.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.476697 10001 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu with pid 10039
07:14:19.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:19.495261 10433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:19.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:19.495862 10433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:19.522 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:14:19.522 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.196.125:44031
07:14:19.523 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
07:14:19.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.465682 10281 master_runner.cc:186] Error getting master registration for 127.9.196.126:37581: OK, instance_id { permanent_uuid: "90f79cce0c894390839c29d841baf254" instance_seqno: 1786259653277796 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:14:19.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.523741 10281 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:14:19.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.523823 10001 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu with pid 10150
07:14:19.552 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:14:19.552 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.196.124:40659
07:14:19.552 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
07:14:19.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.552788 10001 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu with pid 10281
07:14:19.578 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:14:19.578 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.196.126:37581
07:14:19.580 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
07:14:19.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.582264 10001 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:14:19.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:14:19.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/wal
07:14:19.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/data
07:14:19.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:14:19.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:14:19.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:14:19.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:14:19.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/logs
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/data/info.pb
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.196.126:37581
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.196.126
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34735
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kudu_127.9.196.126.keytab
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.196.126
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.196.84:33847
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659
07:14:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:14:19.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33153/jwks.json
07:14:19.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:14:19.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:19.911109 10435 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:14:19.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:19.911711 10435 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:14:19.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:19.912333 10435 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:14:19.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:19.944497 10435 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:14:19.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:19.944713 10435 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:14:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:19.944973 10435 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:14:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:19.945166 10435 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:14:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:19.945374 10435 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:14:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:19.945557 10435 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:14:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:19.945693 10435 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33153/jwks.json
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.984568 10435 master_runner.cc:386] Master server non-default flags:
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.196.84:33847
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/data
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/wal
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kudu_127.9.196.126.keytab
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.196.126
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.196.126:37581
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33153/jwks.json
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/data/info.pb
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.196.126
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34735
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/logs
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:14:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:14:19.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:14:19.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 06:34:24 UTC on ccycloud.quasar-rbacaa.root.comops.site
07:14:19.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575854
07:14:19.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:14:19.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.985689 10435 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:14:19.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:19.986644 10435 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:14:20.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:20.005892 10443 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:21.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:20.005950 10447 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:21.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:20.005947 10441 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:21.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:21.233260 10446 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
07:14:21.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.233423 10435 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:14:21.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.237722 10435 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:14:21.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.240440 10435 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:14:21.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.241878 10435 hybrid_clock.cc:648] HybridClock initialized: now 1786259661241827 us; error 45 us; skew 500 ppm
07:14:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.242753 10435 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:14:21.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:21 dist-test-slave-1nl6 krb5kdc[10016](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786259661, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.196.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:14:21.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.249078 10435 init.cc:377] Logged in from keytab as kudu/127.9.196.126@KRBTEST.COM (short username kudu)
07:14:21.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.255139 10435 webserver.cc:460] Webserver started at http://127.9.196.126:34735/ using document root <none> and password file <none>
07:14:21.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.256292 10435 fs_manager.cc:359] Metadata directory not provided
07:14:21.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.256553 10435 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:14:21.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.266803 10435 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.006s sys 0.001s
07:14:21.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.271647 10459 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:14:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.272787 10435 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
07:14:21.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.273073 10435 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/data,/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/wal
07:14:21.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "90f79cce0c894390839c29d841baf254"
07:14:21.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:14:12 on dist-test-slave-1nl6"
07:14:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.275497 10435 fs_report.cc:383] FS layout report
07:14:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:14:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/wal
07:14:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/wal
07:14:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/data/data
07:14:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:14:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:14:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:14:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:14:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:14:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:14:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:14:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:14:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:14:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:14:21.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:14:21.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.320448 10435 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:14:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.326069 10435 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:14:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.326516 10435 kserver.cc:163] Server-wide thread pool size limit: 3276
07:14:21.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.407760 10435 rpc_server.cc:307] RPC server started. Bound to: 127.9.196.126:37581
07:14:21.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.407929 10536 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.196.126:37581 every 8 connection(s)
07:14:21.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.412052 10435 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/data/info.pb
07:14:21.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.416277 10001 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu as pid 10435
07:14:21.417 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:14:21.417 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.196.125:44031
07:14:21.417 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
07:14:21.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.418483 10001 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:14:21.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:14:21.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/wal
07:14:21.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/data
07:14:21.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:14:21.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:14:21.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:14:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:14:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:14:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:14:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/logs
07:14:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/data/info.pb
07:14:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:14:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:14:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:14:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:14:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:14:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:14:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:14:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:14:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:14:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:14:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:14:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:14:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.196.125:44031
07:14:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.196.125
07:14:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33179
07:14:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kudu_127.9.196.125.keytab
07:14:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.196.125
07:14:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:14:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:14:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:14:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.196.84:33847
07:14:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:14:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:14:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:14:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:14:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659
07:14:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:14:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33153/jwks.json
07:14:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:14:21.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.425175 10537 sys_catalog.cc:263] Verifying existing consensus state
07:14:21.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260809 07:14:21.426062 10537 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:14:21.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.426513 10435 master.cc:537] Master@127.9.196.126:37581 shutting down...
07:14:21.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.508198 10435 master.cc:552] Master@127.9.196.126:37581 shutdown complete.
07:14:21.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:14:21.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:21.767367 10539 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:14:21.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:21.767957 10539 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:14:21.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:21.768636 10539 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:14:21.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:21.802443 10539 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:14:21.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:21.802685 10539 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:14:21.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:21.803035 10539 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:14:21.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:21.803264 10539 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:14:21.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:21.803460 10539 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:14:21.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:21.803656 10539 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:14:21.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:21.803797 10539 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33153/jwks.json
07:14:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.840678 10539 master_runner.cc:386] Master server non-default flags:
07:14:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:14:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.196.84:33847
07:14:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:14:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:14:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/data
07:14:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/wal
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kudu_127.9.196.125.keytab
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.196.125
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.196.125:44031
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33153/jwks.json
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/data/info.pb
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.196.125
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33179
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/logs
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:14:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:14:21.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 06:34:24 UTC on ccycloud.quasar-rbacaa.root.comops.site
07:14:21.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575854
07:14:21.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:14:21.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.842049 10539 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:14:21.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.843477 10539 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:14:21.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:21.863814 10549 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:21.863809 10547 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:21.863989 10546 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:21.866099 10539 server_base.cc:1034] running on GCE node
07:14:23.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.104378 10539 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:14:23.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.107568 10539 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:14:23.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.109018 10539 hybrid_clock.cc:648] HybridClock initialized: now 1786259663108948 us; error 66 us; skew 500 ppm
07:14:23.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.109978 10539 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:14:23.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:23 dist-test-slave-1nl6 krb5kdc[10016](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786259663, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.196.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:14:23.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.116230 10539 init.cc:377] Logged in from keytab as kudu/127.9.196.125@KRBTEST.COM (short username kudu)
07:14:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.121170 10539 webserver.cc:460] Webserver started at http://127.9.196.125:33179/ using document root <none> and password file <none>
07:14:23.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.122351 10539 fs_manager.cc:359] Metadata directory not provided
07:14:23.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.122634 10539 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:14:23.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.133244 10539 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.001s sys 0.004s
07:14:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.138677 10562 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:14:23.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.139983 10539 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
07:14:23.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.140412 10539 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/data,/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/wal
07:14:23.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5d9b55128cc645dda23591f839d83028"
07:14:23.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:14:15 on dist-test-slave-1nl6"
07:14:23.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.143872 10539 fs_report.cc:383] FS layout report
07:14:23.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:14:23.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/wal
07:14:23.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/wal
07:14:23.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/data/data
07:14:23.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:14:23.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:14:23.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:14:23.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:14:23.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:14:23.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:14:23.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:14:23.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:14:23.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:14:23.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:14:23.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:14:23.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.193059 10539 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:14:23.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.199577 10539 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:14:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.200011 10539 kserver.cc:163] Server-wide thread pool size limit: 3276
07:14:23.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.279449 10539 rpc_server.cc:307] RPC server started. Bound to: 127.9.196.125:44031
07:14:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.279603 10643 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.196.125:44031 every 8 connection(s)
07:14:23.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.283205 10539 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/data/info.pb
07:14:23.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.288182 10001 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu as pid 10539
07:14:23.289 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:14:23.289 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.196.124:40659
07:14:23.289 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.291572 10001 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/wal
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/data
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/logs
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/data/info.pb
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.196.124:40659
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.196.124
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38743
07:14:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kudu_127.9.196.124.keytab
07:14:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.196.124
07:14:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:14:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:14:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:14:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.196.84:33847
07:14:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:14:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:14:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:14:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:14:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659
07:14:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:14:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33153/jwks.json
07:14:23.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:14:23.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.296483 10644 sys_catalog.cc:263] Verifying existing consensus state
07:14:23.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260809 07:14:23.297381 10644 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:14:23.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.297984 10539 master.cc:537] Master@127.9.196.125:44031 shutting down...
07:14:23.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.373633 10539 master.cc:552] Master@127.9.196.125:44031 shutdown complete.
07:14:23.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:14:23.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:23.631698 10647 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:14:23.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:23.632395 10647 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:14:23.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:23.633105 10647 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:14:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:23.664847 10647 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:14:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:23.665153 10647 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:14:23.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:23.665566 10647 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:14:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:23.665913 10647 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:14:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:23.666236 10647 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:14:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:23.666556 10647 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:14:23.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:23.666791 10647 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33153/jwks.json
07:14:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.705194 10647 master_runner.cc:386] Master server non-default flags:
07:14:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:14:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.196.84:33847
07:14:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:14:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:14:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/data
07:14:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/wal
07:14:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:14:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/krb5kdc/kudu_127.9.196.124.keytab
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.196.124
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.196.124:40659
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33153/jwks.json
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/data/info.pb
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.196.124
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38743
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/logs
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:14:23.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:14:23.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:14:23.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:14:23.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:14:23.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:14:23.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 06:34:24 UTC on ccycloud.quasar-rbacaa.root.comops.site
07:14:23.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575854
07:14:23.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:14:23.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.706720 10647 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:14:23.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.708132 10647 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:14:23.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:23.726784 10659 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:23.729068 10647 server_base.cc:1034] running on GCE node
07:14:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:23.727025 10660 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:24.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:23.727705 10662 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:14:24.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:24.972692 10647 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:14:24.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:24.976048 10647 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:14:24.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:24.977506 10647 hybrid_clock.cc:648] HybridClock initialized: now 1786259664977466 us; error 62 us; skew 500 ppm
07:14:24.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:24.978331 10647 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:14:24.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:24 dist-test-slave-1nl6 krb5kdc[10016](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786259664, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.196.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:14:24.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:24.987201 10647 init.cc:377] Logged in from keytab as kudu/127.9.196.124@KRBTEST.COM (short username kudu)
07:14:24.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:24.992607 10647 webserver.cc:460] Webserver started at http://127.9.196.124:38743/ using document root <none> and password file <none>
07:14:24.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:24.993489 10647 fs_manager.cc:359] Metadata directory not provided
07:14:24.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:24.993710 10647 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:14:25.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.001998 10647 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.002s sys 0.004s
07:14:25.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.007082 10674 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:14:25.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.008230 10647 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
07:14:25.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.008567 10647 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/data,/tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/wal
07:14:25.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aafc297b81ee4c9ca11d91a1200f3093"
07:14:25.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 07:14:18 on dist-test-slave-1nl6"
07:14:25.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.011116 10647 fs_report.cc:383] FS layout report
07:14:25.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:14:25.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/wal
07:14:25.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/wal
07:14:25.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/data/data
07:14:25.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:14:25.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:14:25.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:14:25.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:14:25.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:14:25.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:14:25.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:14:25.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:14:25.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:14:25.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:14:25.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:14:25.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.068221 10647 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:14:25.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.074262 10647 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:14:25.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.074726 10647 kserver.cc:163] Server-wide thread pool size limit: 3276
07:14:25.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.157608 10647 rpc_server.cc:307] RPC server started. Bound to: 127.9.196.124:40659
07:14:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.157753 10750 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.9.196.124:40659 every 8 connection(s)
07:14:25.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.160651 10647 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskrvi3SN/test-tmp/mini-kudu-cluster5936741127892876289/master-2/data/info.pb
07:14:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.168624 10001 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu as pid 10647
07:14:25.169 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:14:25.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.175216 10751 sys_catalog.cc:263] Verifying existing consensus state
07:14:25.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.182637 10751 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: Bootstrap starting.
07:14:25.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.221704 10751 log.cc:824] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: Log is configured to *not* fsync() on all Append() calls
07:14:25.223 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.226387 10751 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
07:14:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.227350 10751 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: Bootstrap complete.
07:14:25.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.256703 10751 raft_consensus.cc:348] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "90f79cce0c894390839c29d841baf254" member_type: VOTER last_known_addr { host: "127.9.196.126" port: 37581 } } peers { permanent_uuid: "5d9b55128cc645dda23591f839d83028" member_type: VOTER last_known_addr { host: "127.9.196.125" port: 44031 } } peers { permanent_uuid: "aafc297b81ee4c9ca11d91a1200f3093" member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } }
07:14:25.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.260018 10751 raft_consensus.cc:374] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:14:25.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.260325 10751 raft_consensus.cc:729] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: aafc297b81ee4c9ca11d91a1200f3093, State: Initialized, Role: FOLLOWER
07:14:25.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.261523 10751 consensus_queue.cc:260] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "90f79cce0c894390839c29d841baf254" member_type: VOTER last_known_addr { host: "127.9.196.126" port: 37581 } } peers { permanent_uuid: "5d9b55128cc645dda23591f839d83028" member_type: VOTER last_known_addr { host: "127.9.196.125" port: 44031 } } peers { permanent_uuid: "aafc297b81ee4c9ca11d91a1200f3093" member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } }
07:14:25.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.284042 10756 sys_catalog.cc:455] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "90f79cce0c894390839c29d841baf254" member_type: VOTER last_known_addr { host: "127.9.196.126" port: 37581 } } peers { permanent_uuid: "5d9b55128cc645dda23591f839d83028" member_type: VOTER last_known_addr { host: "127.9.196.125" port: 44031 } } peers { permanent_uuid: "aafc297b81ee4c9ca11d91a1200f3093" member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } } }
07:14:25.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.284999 10756 sys_catalog.cc:458] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [sys.catalog]: This master's current role is: FOLLOWER
07:14:25.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.285946 10751 sys_catalog.cc:564] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [sys.catalog]: configured and running, proceeding with master startup.
07:14:25.297 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:25.303 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.196.124:40659(127.9.196.124:40659)] connecting to peer
07:14:25.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:25.330770 10683 tablet.cc:2369] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:14:25.341 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:25.342 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:25.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:25.345 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:25.346 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:25.347 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:25.348 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.9.196.124:40659(127.9.196.124:40659)] Successfully connected to peer
07:14:25.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:25.367250 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:25.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:25.371030 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:25.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.334105 10647 master_runner.cc:186] Error getting master registration for 127.9.196.126:37581: Network error: Client connection negotiation failed: client connection to 127.9.196.126:37581: connect: Connection refused (error 111),
07:14:25.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.383155 10647 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:14:25.441 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
07:14:25.450 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.9.196.124
07:14:25.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.565588 10756 raft_consensus.cc:482] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:14:25.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.566232 10756 raft_consensus.cc:504] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "90f79cce0c894390839c29d841baf254" member_type: VOTER last_known_addr { host: "127.9.196.126" port: 37581 } } peers { permanent_uuid: "5d9b55128cc645dda23591f839d83028" member_type: VOTER last_known_addr { host: "127.9.196.125" port: 44031 } } peers { permanent_uuid: "aafc297b81ee4c9ca11d91a1200f3093" member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } }
07:14:25.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:25.572806 10678 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.196.126:37581: connect: Connection refused (error 111)
07:14:25.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.574843 10756 leader_election.cc:288] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 90f79cce0c894390839c29d841baf254 (127.9.196.126:37581), 5d9b55128cc645dda23591f839d83028 (127.9.196.125:44031)
07:14:25.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:25.584465 10678 leader_election.cc:334] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 90f79cce0c894390839c29d841baf254 (127.9.196.126:37581): Network error: Client connection negotiation failed: client connection to 127.9.196.126:37581: connect: Connection refused (error 111)
07:14:25.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:25.609630 10677 leader_election.cc:334] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5d9b55128cc645dda23591f839d83028 (127.9.196.125:44031): Network error: Client connection negotiation failed: client connection to 127.9.196.125:44031: connect: Connection refused (error 111)
07:14:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.610687 10677 leader_election.cc:302] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: aafc297b81ee4c9ca11d91a1200f3093; no voters: 5d9b55128cc645dda23591f839d83028, 90f79cce0c894390839c29d841baf254
07:14:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:25.611583 10756 raft_consensus.cc:2726] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:14:25.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:25 dist-test-slave-1nl6 krb5kdc[10016](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786259659, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.196.124@KRBTEST.COM
07:14:25.718 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x8c29fdb0, L:/127.0.0.1:47432 - R:/127.9.196.124:40659] using SASL/GSSAPI
07:14:25.736 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:25.756 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:25.759 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:25.760 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:25.762 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:25.763 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:25.765 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:25.765 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:25.766 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:25.768 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:25.775 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:25.778 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:25.779 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:25.780 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:25.780 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:25.782 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:25.783 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:25.783 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:25.789 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:25.815 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:25.817 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:25.818 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:25.818 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:25.819 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:25.821 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:25.821 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:25.821 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:25.825 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:25.855 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:25.856 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:25.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:25.859 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:25.859 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:25.861 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:25.862 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:25.862 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:25.864 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:25.896 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:25.897 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:25.899 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:25.899 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:25.900 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:25.903 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:25.903 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:25.904 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:25.905 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:25.955 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:25.958 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:25.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:25.958 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:25.960 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:25.965 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:25.965 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:25.966 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:25.967 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:26.036 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:26.038 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:26.039 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:26.039 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:26.040 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:26.041 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:26.041 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:26.045 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:26.047 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:26.096 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:26.097 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:26.101 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:26.101 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:26.105 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:26.106 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:26.107 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:26.107 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:26.112 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:26.135 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:26.136 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:26.144 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:26.144 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:26.145 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:26.145 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:26.146 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:26.146 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:26.148 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:26.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:26.375385 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:26.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:26.376000 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:26.995 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:26.996 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:26.997 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:26.997 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:26.997 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:26.999 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:26.999 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:26.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:27.003 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:27.186038 10789 raft_consensus.cc:482] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:14:27.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:27.186609 10789 raft_consensus.cc:504] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "90f79cce0c894390839c29d841baf254" member_type: VOTER last_known_addr { host: "127.9.196.126" port: 37581 } } peers { permanent_uuid: "5d9b55128cc645dda23591f839d83028" member_type: VOTER last_known_addr { host: "127.9.196.125" port: 44031 } } peers { permanent_uuid: "aafc297b81ee4c9ca11d91a1200f3093" member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } }
07:14:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:27.188565 10789 leader_election.cc:288] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 90f79cce0c894390839c29d841baf254 (127.9.196.126:37581), 5d9b55128cc645dda23591f839d83028 (127.9.196.125:44031)
07:14:27.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:27.195415 10677 leader_election.cc:334] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5d9b55128cc645dda23591f839d83028 (127.9.196.125:44031): Network error: Client connection negotiation failed: client connection to 127.9.196.125:44031: connect: Connection refused (error 111)
07:14:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:27.195928 10678 leader_election.cc:334] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 90f79cce0c894390839c29d841baf254 (127.9.196.126:37581): Network error: Client connection negotiation failed: client connection to 127.9.196.126:37581: connect: Connection refused (error 111)
07:14:27.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:27.196539 10678 leader_election.cc:302] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: aafc297b81ee4c9ca11d91a1200f3093; no voters: 5d9b55128cc645dda23591f839d83028, 90f79cce0c894390839c29d841baf254
07:14:27.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:27.197338 10789 raft_consensus.cc:2726] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:14:27.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:27.379311 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:27.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:27.379658 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:27.815 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:27.816 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:27.817 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:27.817 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:27.818 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:27.820 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:27.820 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:27.821 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:27.822 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:28.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:28.383587 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:28.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:28.384305 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:29.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:29.388545 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:29.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:29.389204 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:29.701647 10797 raft_consensus.cc:482] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:14:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:29.702085 10797 raft_consensus.cc:504] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "90f79cce0c894390839c29d841baf254" member_type: VOTER last_known_addr { host: "127.9.196.126" port: 37581 } } peers { permanent_uuid: "5d9b55128cc645dda23591f839d83028" member_type: VOTER last_known_addr { host: "127.9.196.125" port: 44031 } } peers { permanent_uuid: "aafc297b81ee4c9ca11d91a1200f3093" member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } }
07:14:29.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:29.703609 10797 leader_election.cc:288] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 90f79cce0c894390839c29d841baf254 (127.9.196.126:37581), 5d9b55128cc645dda23591f839d83028 (127.9.196.125:44031)
07:14:29.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:29.708501 10677 leader_election.cc:334] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5d9b55128cc645dda23591f839d83028 (127.9.196.125:44031): Network error: Client connection negotiation failed: client connection to 127.9.196.125:44031: connect: Connection refused (error 111)
07:14:29.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:29.708884 10678 leader_election.cc:334] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 90f79cce0c894390839c29d841baf254 (127.9.196.126:37581): Network error: Client connection negotiation failed: client connection to 127.9.196.126:37581: connect: Connection refused (error 111)
07:14:29.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:29.709273 10678 leader_election.cc:302] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: aafc297b81ee4c9ca11d91a1200f3093; no voters: 5d9b55128cc645dda23591f839d83028, 90f79cce0c894390839c29d841baf254
07:14:29.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:29.709969 10797 raft_consensus.cc:2726] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:14:30.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:30.389037 10647 master_runner.cc:186] Error getting master registration for 127.9.196.126:37581: Network error: Client connection negotiation failed: client connection to 127.9.196.126:37581: connect: Connection refused (error 111),
07:14:30.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:30.389523 10647 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:14:30.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:30.396937 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:30.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:30.397363 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:31.035 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:31.036 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:31.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:31.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:31.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:31.039 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:31.039 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:31.039 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:31.041 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:31.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:31.400934 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:31.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:31.401280 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:31.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:31.428201 9789 debug-util.cc:398] Leaking SignalData structure 0x7b08000af060 after lost signal to thread 9603
07:14:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:31.429514 9789 debug-util.cc:398] Leaking SignalData structure 0x7b08000b4620 after lost signal to thread 9793
07:14:31.435 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:31.436 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:31.440 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:31.440 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:31.440 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:31.442 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:31.442 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:31.442 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:31.445 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:31.955 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:31.956 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:31.957 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:31.958 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:31.958 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:31.960 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:31.961 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:31.961 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:31.963 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:32.255 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:32.256 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:32.260 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:32.260 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:32.260 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:32.261 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:32.262 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:32.262 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:32.264 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:32.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:32.404843 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:32.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:32.405153 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:32.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:32.914752 10807 raft_consensus.cc:482] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:14:32.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:32.915263 10807 raft_consensus.cc:504] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "90f79cce0c894390839c29d841baf254" member_type: VOTER last_known_addr { host: "127.9.196.126" port: 37581 } } peers { permanent_uuid: "5d9b55128cc645dda23591f839d83028" member_type: VOTER last_known_addr { host: "127.9.196.125" port: 44031 } } peers { permanent_uuid: "aafc297b81ee4c9ca11d91a1200f3093" member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } }
07:14:32.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:32.918699 10807 leader_election.cc:288] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 90f79cce0c894390839c29d841baf254 (127.9.196.126:37581), 5d9b55128cc645dda23591f839d83028 (127.9.196.125:44031)
07:14:32.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:32.920073 10678 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.196.126:37581: connect: Connection refused (error 111) [suppressed 5 similar messages]
07:14:32.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:32.922832 10677 leader_election.cc:334] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5d9b55128cc645dda23591f839d83028 (127.9.196.125:44031): Network error: Client connection negotiation failed: client connection to 127.9.196.125:44031: connect: Connection refused (error 111)
07:14:32.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:32.924229 10678 leader_election.cc:334] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 90f79cce0c894390839c29d841baf254 (127.9.196.126:37581): Network error: Client connection negotiation failed: client connection to 127.9.196.126:37581: connect: Connection refused (error 111)
07:14:32.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:32.924666 10678 leader_election.cc:302] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: aafc297b81ee4c9ca11d91a1200f3093; no voters: 5d9b55128cc645dda23591f839d83028, 90f79cce0c894390839c29d841baf254
07:14:32.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:32.925329 10807 raft_consensus.cc:2726] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:14:33.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:33.408587 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:33.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:33.408946 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:34.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:34.413229 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:34.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:34.413564 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:34.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:34.484510 10820 raft_consensus.cc:482] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:14:34.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:34.484992 10820 raft_consensus.cc:504] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "90f79cce0c894390839c29d841baf254" member_type: VOTER last_known_addr { host: "127.9.196.126" port: 37581 } } peers { permanent_uuid: "5d9b55128cc645dda23591f839d83028" member_type: VOTER last_known_addr { host: "127.9.196.125" port: 44031 } } peers { permanent_uuid: "aafc297b81ee4c9ca11d91a1200f3093" member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } }
07:14:34.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:34.486852 10820 leader_election.cc:288] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 90f79cce0c894390839c29d841baf254 (127.9.196.126:37581), 5d9b55128cc645dda23591f839d83028 (127.9.196.125:44031)
07:14:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:34.491765 10677 leader_election.cc:334] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5d9b55128cc645dda23591f839d83028 (127.9.196.125:44031): Network error: Client connection negotiation failed: client connection to 127.9.196.125:44031: connect: Connection refused (error 111)
07:14:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:34.492280 10678 leader_election.cc:334] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 90f79cce0c894390839c29d841baf254 (127.9.196.126:37581): Network error: Client connection negotiation failed: client connection to 127.9.196.126:37581: connect: Connection refused (error 111)
07:14:34.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:34.492581 10678 leader_election.cc:302] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: aafc297b81ee4c9ca11d91a1200f3093; no voters: 5d9b55128cc645dda23591f839d83028, 90f79cce0c894390839c29d841baf254
07:14:34.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:34.493150 10820 raft_consensus.cc:2726] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:14:34.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:34 dist-test-slave-1nl6 krb5kdc[10016](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786259674, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.196.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:14:34.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:34.991744 10670 init.cc:295] Successfully reacquired a new kerberos TGT
07:14:35.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:35.392520 10647 master_runner.cc:186] Error getting master registration for 127.9.196.126:37581: Network error: Client connection negotiation failed: client connection to 127.9.196.126:37581: connect: Connection refused (error 111),
07:14:35.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:35.392932 10647 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:14:35.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:35.416888 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:35.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:35.417147 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:36.115 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:36.116 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:36.117 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:36.117 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:36.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:36.121 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:36.121 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:36.121 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:36.123 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:36.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:36.420050 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:36.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:36.420305 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:37.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:37.401088 9981 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6780 after lost signal to thread 9810
07:14:37.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:37.402988 9981 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6880 after lost signal to thread 9985
07:14:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:37.427024 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:37.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:37.427536 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:38.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:38.544248 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:38.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:38.554802 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:39.016 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:39.018 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:39.019 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:39.019 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:39.019 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:39.022 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:39.022 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:39.022 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:39.026 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:39.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:39.331545 10839 raft_consensus.cc:482] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:14:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:39.332129 10839 raft_consensus.cc:504] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "90f79cce0c894390839c29d841baf254" member_type: VOTER last_known_addr { host: "127.9.196.126" port: 37581 } } peers { permanent_uuid: "5d9b55128cc645dda23591f839d83028" member_type: VOTER last_known_addr { host: "127.9.196.125" port: 44031 } } peers { permanent_uuid: "aafc297b81ee4c9ca11d91a1200f3093" member_type: VOTER last_known_addr { host: "127.9.196.124" port: 40659 } }
07:14:39.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:39.335534 10839 leader_election.cc:288] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 90f79cce0c894390839c29d841baf254 (127.9.196.126:37581), 5d9b55128cc645dda23591f839d83028 (127.9.196.125:44031)
07:14:39.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:39.338668 10678 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.196.126:37581: connect: Connection refused (error 111) [suppressed 3 similar messages]
07:14:39.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:39.343776 10677 leader_election.cc:334] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 5d9b55128cc645dda23591f839d83028 (127.9.196.125:44031): Network error: Client connection negotiation failed: client connection to 127.9.196.125:44031: connect: Connection refused (error 111)
07:14:39.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:39.344645 10678 leader_election.cc:334] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 90f79cce0c894390839c29d841baf254 (127.9.196.126:37581): Network error: Client connection negotiation failed: client connection to 127.9.196.126:37581: connect: Connection refused (error 111)
07:14:39.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:39.345227 10678 leader_election.cc:302] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: aafc297b81ee4c9ca11d91a1200f3093; no voters: 5d9b55128cc645dda23591f839d83028, 90f79cce0c894390839c29d841baf254
07:14:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:39.346333 10839 raft_consensus.cc:2726] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:14:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:39.566828 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:39.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:39.567291 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:40.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:40.395635 10647 master_runner.cc:186] Error getting master registration for 127.9.196.126:37581: Network error: Client connection negotiation failed: client connection to 127.9.196.126:37581: connect: Connection refused (error 111),
07:14:40.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:40.396029 10647 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:14:40.435 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:40.436 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:40.437 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:40.438 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:40.438 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:40.439 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:40.439 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:40.439 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:40.441 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:40.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:40.570222 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:40.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:40.570492 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:41.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:41.573423 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:41.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:41.573683 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:42.135 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:42.136 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:42.137 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:42.137 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:42.137 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:42.141 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:42.141 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:42.141 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:42.142 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:42.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:42.576684 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:42.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:42.577010 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:43.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:43.579839 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:43.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:43.580080 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:44.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:44.583503 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:44.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:44.583845 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:44.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 09 07:14:44 dist-test-slave-1nl6 krb5kdc[10016](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786259684, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.196.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:14:44.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:44.994333 10670 init.cc:295] Successfully reacquired a new kerberos TGT
07:14:45.335 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:45.336 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:45.336 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:45.336 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:45.337 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:45.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:45.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:45.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:45.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:45.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:45.399514 10647 master_runner.cc:186] Error getting master registration for 127.9.196.126:37581: Network error: Client connection negotiation failed: client connection to 127.9.196.126:37581: connect: Connection refused (error 111),
07:14:45.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:45.399917 10647 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:14:45.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:45.586658 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:45.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:45.586957 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:46.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:46.589835 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:46.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:46.590090 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:47.275 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:47.276 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:47.276 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:47.276 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:47.277 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:47.277 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:47.277 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:47.278 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:47.280 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:47.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:47.592890 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:47.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:47.593158 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:48.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:48.596239 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:48.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:48.596575 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:48.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:48.768173 9298 debug-util.cc:398] Leaking SignalData structure 0x7b08000b3100 after lost signal to thread 9193
07:14:48.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:48.768903 9298 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5460 after lost signal to thread 9302
07:14:49.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:49.599808 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:49.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:49.600097 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:50.315 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:50.316 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:50.316 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:50.317 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:50.317 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:50.319 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:50.319 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:50.319 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:50.321 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:50.402670 10647 master_runner.cc:186] Error getting master registration for 127.9.196.126:37581: Network error: Client connection negotiation failed: client connection to 127.9.196.126:37581: connect: Connection refused (error 111),
07:14:50.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:50.403201 10647 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:14:50.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:50.603609 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:50.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:50.603942 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:51.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:51.606719 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:51.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:51.607055 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:52.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:52.610028 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:52.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:52.610368 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:53.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:53.613251 10779 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aafc297b81ee4c9ca11d91a1200f3093: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:14:53.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 07:14:53.613554 10779 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:14:53.795 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.126:37581(127.9.196.126:37581)] connecting to peer
07:14:53.796 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.9.196.125:44031(127.9.196.125:44031)] connecting to peer
07:14:53.797 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:53.797 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.196.126:37581(127.9.196.126:37581)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:53.797 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.126:37581: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581
07:14:53.798 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:53.798 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.125:44031(127.9.196.125:44031)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:53.798 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.196.125:44031: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
07:14:53.802 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659; will retry
07:14:53.808 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28629), Traces: [0ms] refreshing cache from master, [34ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [163ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [165ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [551ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [557ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [583ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [585ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [587ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [588ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [594ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [600ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [603ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [608ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [609ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [639ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [641ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [643ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [674ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [679ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [682ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [683ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [715ms] refreshing cache from master, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [720ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [723ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [725ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [774ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [778ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [784ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [785ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [787ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [860ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [862ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [866ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [867ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [915ms] refreshing cache from master, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [921ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [926ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [927ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [932ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [954ms] refreshing cache from master, [955ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [964ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [965ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [967ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [968ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [1814ms] refreshing cache from master, [1814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [1816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [1817ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [1819ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [1819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [1823ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [1823ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [2635ms] refreshing cache from master, [2635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [2636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [2637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [2638ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [2640ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [2642ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [2642ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [5855ms] refreshing cache from master, [5855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [5856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [5857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [5858ms] too many traces: truncated at 100 traces, deferred=Deferred@1246642991(state=PENDING, result=null, callback=(continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995) -> (continuation of Deferred@2142249098 after retry RPC after error@259758357) -> (continuation of Deferred@1107721454 after retry RPC after error@732290621) -> (continuation of Deferred@1441734579 after retry RPC after error@110069546) -> (continuation of Deferred@1998547481 after retry RPC after error@1917716209), errback=(continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995) -> (continuation of Deferred@2142249098 after retry RPC after error@259758357) -> (continuation of Deferred@1107721454 after retry RPC after error@732290621) -> (continuation of Deferred@1441734579 after retry RPC after error@110069546) -> (continuation of Deferred@1998547481 after retry RPC after error@1917716209)))
07:14:53.814 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.9.196.126:37581(127.9.196.126:37581)] disconnecting while in state TERMINATED
07:14:53.816 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.9.196.124:40659(127.9.196.124:40659)] disconnecting while in state READY
07:14:53.816 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.9.196.125:44031(127.9.196.125:44031)] disconnecting while in state TERMINATED
07:14:53.822 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:14:53.824 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.9.196.124:40659(127.9.196.124:40659)] handling channelInactive
07:14:53.824 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:14:53.824 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.196.124:40659(127.9.196.124:40659)] cleaning up while in state READY due to: connection closed
07:14:53.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:53.828390 10001 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu with pid 10647
07:14:55.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T07:14:55Z chronyd exiting
07:14:56.092 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:14:56.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:56.093489 8952 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu with pid 9366
07:14:56.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:56.150444 8952 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu with pid 9601
07:14:56.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:56.193893 8952 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu with pid 9805
07:14:56.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:56.231515 8952 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu with pid 8967
07:14:56.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:56.281615 8952 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu with pid 9069
07:14:56.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 07:14:56.321455 8952 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskrvi3SN/build/tsan/bin/kudu with pid 9185
07:14:57.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T07:14:57Z chronyd exiting
07:14:58.524 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28629), Traces: [0ms] refreshing cache from master, [34ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [163ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [165ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [551ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [557ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [583ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [585ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [587ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [588ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [594ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [600ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [603ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [608ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [609ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [639ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [641ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [643ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [674ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [679ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [682ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [683ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [715ms] refreshing cache from master, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [720ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [723ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [725ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [774ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [778ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [784ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [785ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [787ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [860ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [862ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [866ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [867ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [915ms] refreshing cache from master, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [921ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [926ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [927ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [932ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [954ms] refreshing cache from master, [955ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [964ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [965ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [967ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [968ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [1814ms] refreshing cache from master, [1814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [1816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [1817ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [1819ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [1819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [1823ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [1823ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [2635ms] refreshing cache from master, [2635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [2636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [2637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [2638ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [2640ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [2642ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [2642ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [5855ms] refreshing cache from master, [5855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [5856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [5857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [5858ms] too many traces: truncated at 100 traces, deferred=Deferred@1246642991(state=PENDING, result=null, callback=(continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995) -> (continuation of Deferred@2142249098 after retry RPC after error@259758357) -> (continuation of Deferred@1107721454 after retry RPC after error@732290621) -> (continuation of Deferred@1441734579 after retry RPC after error@110069546) -> (continuation of Deferred@1998547481 after retry RPC after error@1917716209), errback=(continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995) -> (continuation of Deferred@2142249098 after retry RPC after error@259758357) -> (continuation of Deferred@1107721454 after retry RPC after error@732290621) -> (continuation of Deferred@1441734579 after retry RPC after error@110069546) -> (continuation of Deferred@1998547481 after retry RPC after error@1917716209)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.9.196.125:44031
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28629), Traces: [0ms] refreshing cache from master, [34ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [163ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [165ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [551ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [557ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [583ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [585ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [587ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [588ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [594ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [600ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [603ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [608ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [609ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [639ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [641ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [643ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [674ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [679ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [682ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [683ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [715ms] refreshing cache from master, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [720ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [723ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [725ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [774ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [778ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [784ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [785ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [787ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [860ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [862ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [866ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [867ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [915ms] refreshing cache from master, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [921ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [926ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [927ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [932ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [954ms] refreshing cache from master, [955ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [964ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [965ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [967ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [968ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [1814ms] refreshing cache from master, [1814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [1816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [1817ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [1819ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [1819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [1823ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [1823ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [2635ms] refreshing cache from master, [2635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [2636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [2637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [2638ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [2640ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [2642ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [2642ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [5855ms] refreshing cache from master, [5855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [5856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [5857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [5858ms] too many traces: truncated at 100 traces, deferred=Deferred@1246642991(state=PENDING, result=null, callback=(continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995) -> (continuation of Deferred@2142249098 after retry RPC after error@259758357) -> (continuation of Deferred@1107721454 after retry RPC after error@732290621) -> (continuation of Deferred@1441734579 after retry RPC after error@110069546) -> (continuation of Deferred@1998547481 after retry RPC after error@1917716209), errback=(continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995) -> (continuation of Deferred@2142249098 after retry RPC after error@259758357) -> (continuation of Deferred@1107721454 after retry RPC after error@732290621) -> (continuation of Deferred@1441734579 after retry RPC after error@110069546) -> (continuation of Deferred@1998547481 after retry RPC after error@1917716209)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.9.196.125:44031
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
07:14:59.684 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
07:14:59.684 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28629), Traces: [0ms] refreshing cache from master, [34ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [163ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [165ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [551ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [557ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [583ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [585ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [587ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [588ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [594ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [600ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [603ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [608ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [609ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [639ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [641ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [643ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [674ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [679ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [682ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [683ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [715ms] refreshing cache from master, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [720ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [723ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [725ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [774ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [778ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [784ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [785ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [787ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [860ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [862ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [866ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [867ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [915ms] refreshing cache from master, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [921ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [926ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [927ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [932ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [954ms] refreshing cache from master, [955ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [964ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [965ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [967ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [968ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [1814ms] refreshing cache from master, [1814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [1816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [1817ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [1819ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [1819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [1823ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [1823ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [2635ms] refreshing cache from master, [2635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [2636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [2637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [2638ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [2640ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [2642ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [2642ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [5855ms] refreshing cache from master, [5855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [5856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [5857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [5858ms] too many traces: truncated at 100 traces, deferred=Deferred@1246642991(state=PENDING, result=null, callback=(continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995) -> (continuation of Deferred@2142249098 after retry RPC after error@259758357) -> (continuation of Deferred@1107721454 after retry RPC after error@732290621) -> (continuation of Deferred@1441734579 after retry RPC after error@110069546) -> (continuation of Deferred@1998547481 after retry RPC after error@1917716209), errback=(continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995) -> (continuation of Deferred@2142249098 after retry RPC after error@259758357) -> (continuation of Deferred@1107721454 after retry RPC after error@732290621) -> (continuation of Deferred@1441734579 after retry RPC after error@110069546) -> (continuation of Deferred@1998547481 after retry RPC after error@1917716209)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.9.196.125:44031
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=28629), Traces: [0ms] refreshing cache from master, [34ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [163ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [165ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [551ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [557ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [583ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [585ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [587ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [588ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [594ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [600ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [603ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [608ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [609ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [639ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [641ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [643ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [674ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [679ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [682ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [683ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [715ms] refreshing cache from master, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [720ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [723ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [725ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [726ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [774ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [778ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [784ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [785ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [787ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [860ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [862ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [866ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [867ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [915ms] refreshing cache from master, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [919ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [921ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [926ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [927ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [932ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [954ms] refreshing cache from master, [955ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [964ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [965ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [967ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [968ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [1814ms] refreshing cache from master, [1814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [1816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [1817ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [1819ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [1819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [1823ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [1823ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [2635ms] refreshing cache from master, [2635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [2636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [2637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [2638ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.126:37581: Network error: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581, [2640ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.125:44031: Network error: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [2642ms] Sub RPC ConnectToMaster: received response from server master-127.9.196.124:40659: OK, [2642ms] delaying RPC due to: Service unavailable: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031, [5855ms] refreshing cache from master, [5855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.126:37581, [5856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.125:44031, [5857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.196.124:40659, [5858ms] too many traces: truncated at 100 traces, deferred=Deferred@1246642991(state=PENDING, result=null, callback=(continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995) -> (continuation of Deferred@2142249098 after retry RPC after error@259758357) -> (continuation of Deferred@1107721454 after retry RPC after error@732290621) -> (continuation of Deferred@1441734579 after retry RPC after error@110069546) -> (continuation of Deferred@1998547481 after retry RPC after error@1917716209), errback=(continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995) -> (continuation of Deferred@2142249098 after retry RPC after error@259758357) -> (continuation of Deferred@1107721454 after retry RPC after error@732290621) -> (continuation of Deferred@1441734579 after retry RPC after error@110069546) -> (continuation of Deferred@1998547481 after retry RPC after error@1917716209)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.9.196.126:37581,127.9.196.125:44031,127.9.196.124:40659) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.126:37581(127.9.196.126:37581): Connection refused: /127.9.196.126:37581,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.9.196.125:44031(127.9.196.125:44031): Connection refused: /127.9.196.125:44031
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.9.196.125:44031
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)