19:31:09.159 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
19:31:09.173 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskDtkeQs/test-tmp
19:31:09.176 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
19:31:09.179 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskDtkeQs/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
19:31:09.489 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
19:31:09.639 [DEBUG - main] (MiniKuduCluster.java:195) Response:
19:31:09.640 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
19:31:09.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:09.643801 2552 mini_oidc.cc:152] Starting JWKS server
19:31:09.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:09.647632 2552 webserver.cc:460] Webserver started at http://127.0.0.1:43327/ using document root <none> and password file <none>
19:31:09.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:09.648178 2552 mini_oidc.cc:189] Starting OIDC Discovery server
19:31:09.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:09.650583 2552 webserver.cc:460] Webserver started at http://127.0.0.1:33761/ using document root <none> and password file <none>
19:31:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T19:31:09Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
19:31:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T19:31:09Z Disabled control of system clock
19:31:09.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:09.702176 2552 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:09.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:09.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-0/wal
19:31:09.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-0/data
19:31:09.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:31:09.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:31:09.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:31:09.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:09.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:09.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:09.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-0/logs
19:31:09.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-0/data/info.pb
19:31:09.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:09.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:31:09.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:31:09.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:31:09.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:31:09.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:09.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:09.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:09.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:31:09.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:31:09.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:31:09.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:31:09.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.126.62:36437
19:31:09.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.126.62
19:31:09.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:31:09.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.126.20:42927
19:31:09.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:09.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:09.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:31:09.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:31:09.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.126.62:36437,127.2.126.61:37711,127.2.126.60:39801
19:31:09.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:31:09.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43327/jwks.json with env {}
19:31:10.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:10.023947 2564 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:31:10.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:10.024926 2564 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:31:10.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:10.025784 2564 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:31:10.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:10.061410 2564 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:31:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:10.061807 2564 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:31:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:10.062121 2564 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:31:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:10.062448 2564 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:31:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:10.062812 2564 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:31:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:10.063074 2564 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43327/jwks.json
19:31:10.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:10.099115 2564 master_runner.cc:386] Master server non-default flags:
19:31:10.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:10.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.126.20:42927
19:31:10.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:10.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:31:10.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-0/data
19:31:10.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-0/wal
19:31:10.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:31:10.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.126.62:36437,127.2.126.61:37711,127.2.126.60:39801
19:31:10.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:10.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:10.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:31:10.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.126.62:36437
19:31:10.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:31:10.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:31:10.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:31:10.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43327/jwks.json
19:31:10.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:10.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:10.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-0/data/info.pb
19:31:10.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.126.62
19:31:10.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:31:10.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:31:10.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:10.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:31:10.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:31:10.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:10.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-0/logs
19:31:10.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:10.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:31:10.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:31:10.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:31:10.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:31:10.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:31:10.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:31:10.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:31:10.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:31:10.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:31:10.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:10.100982 2564 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:31:10.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:10.103298 2564 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:31:10.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:10.124563 2571 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:11.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:10.124869 2573 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:11.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:10.126554 2564 server_base.cc:1034] running on GCE node
19:31:11.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:10.124872 2570 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:11.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.413108 2564 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:31:11.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.416081 2564 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:31:11.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.417500 2564 hybrid_clock.cc:648] HybridClock initialized: now 1790623871417473 us; error 62 us; skew 500 ppm
19:31:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.418439 2564 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:31:11.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.427249 2564 webserver.cc:460] Webserver started at http://127.2.126.62:39111/ using document root <none> and password file <none>
19:31:11.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.428308 2564 fs_manager.cc:359] Metadata directory not provided
19:31:11.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.428505 2564 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:31:11.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.428894 2564 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:31:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.434302 2564 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-0/data/instance:
19:31:11.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e5f9ceb87f6146dd8c7532c66ed0de44"
19:31:11.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:11 on dist-test-slave-fr74"
19:31:11.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.435590 2564 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-0/wal/instance:
19:31:11.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e5f9ceb87f6146dd8c7532c66ed0de44"
19:31:11.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:11 on dist-test-slave-fr74"
19:31:11.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.443856 2564 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.000s
19:31:11.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.449851 2580 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:31:11.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.451316 2564 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
19:31:11.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.451769 2564 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-0/data,/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-0/wal
19:31:11.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e5f9ceb87f6146dd8c7532c66ed0de44"
19:31:11.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:11 on dist-test-slave-fr74"
19:31:11.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.452211 2564 fs_report.cc:383] FS layout report
19:31:11.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:31:11.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-0/wal
19:31:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-0/wal
19:31:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-0/data/data
19:31:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:31:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:31:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:31:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:31:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:31:11.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:31:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:31:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:31:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:31:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:31:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:31:11.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.488251 2564 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:31:11.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.490224 2564 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:31:11.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.490825 2564 kserver.cc:163] Server-wide thread pool size limit: 3276
19:31:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.575070 2564 rpc_server.cc:307] RPC server started. Bound to: 127.2.126.62:36437
19:31:11.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.575125 2631 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.126.62:36437 every 8 connection(s)
19:31:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.577714 2564 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-0/data/info.pb
19:31:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.580786 2552 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu as pid 2564
19:31:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.582048 2552 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-0/wal/instance
19:31:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.584167 2632 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:31:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.585908 2552 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-1/wal
19:31:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-1/data
19:31:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:31:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:31:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:31:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-1/logs
19:31:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-1/data/info.pb
19:31:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:31:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:31:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:31:11.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:31:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:31:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:31:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:31:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:31:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.126.61:37711
19:31:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.126.61
19:31:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:31:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.126.20:42927
19:31:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:31:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:31:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.126.62:36437,127.2.126.61:37711,127.2.126.60:39801
19:31:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:31:11.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43327/jwks.json with env {}
19:31:11.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.600198 2632 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 } has no permanent_uuid. Determining permanent_uuid...
19:31:11.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.636264 2632 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } has no permanent_uuid. Determining permanent_uuid...
19:31:11.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:11.638371 2581 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.126.61:37711: connect: Connection refused (error 111)
19:31:11.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:11.643369 2632 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.126.61:37711: Network error: Client connection negotiation failed: client connection to 127.2.126.61:37711: connect: Connection refused (error 111)
19:31:11.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.693074 2632 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } attempt: 1
19:31:11.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:11.697332 2632 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.126.61:37711: Network error: Client connection negotiation failed: client connection to 127.2.126.61:37711: connect: Connection refused (error 111)
19:31:11.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.765976 2632 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } attempt: 2
19:31:11.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:11.770076 2632 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.126.61:37711: Network error: Client connection negotiation failed: client connection to 127.2.126.61:37711: connect: Connection refused (error 111)
19:31:11.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:11.861557 2632 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } attempt: 3
19:31:11.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:11.865716 2632 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.126.61:37711: Network error: Client connection negotiation failed: client connection to 127.2.126.61:37711: connect: Connection refused (error 111)
19:31:11.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:11.923067 2634 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:31:11.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:11.923816 2634 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:31:11.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:11.924492 2634 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:31:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:11.962100 2634 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:31:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:11.962390 2634 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:31:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:11.962694 2634 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:31:11.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:11.962930 2634 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:31:11.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:11.963116 2634 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:31:11.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:11.963464 2634 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43327/jwks.json
19:31:12.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:12.005415 2634 master_runner.cc:386] Master server non-default flags:
19:31:12.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:12.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.126.20:42927
19:31:12.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:12.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:31:12.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-1/data
19:31:12.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-1/wal
19:31:12.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:31:12.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.126.62:36437,127.2.126.61:37711,127.2.126.60:39801
19:31:12.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:12.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:12.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:31:12.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.126.61:37711
19:31:12.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:31:12.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:31:12.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:31:12.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43327/jwks.json
19:31:12.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:12.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:12.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-1/data/info.pb
19:31:12.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.126.61
19:31:12.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:31:12.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:31:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:31:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:31:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-1/logs
19:31:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:31:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:31:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:31:12.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:31:12.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:31:12.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:31:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:31:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:31:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:31:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:12.006628 2634 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:31:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:12.008476 2634 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:31:12.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:12.009194 2632 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } attempt: 4
19:31:12.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:12.014230 2632 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.126.61:37711: Network error: Client connection negotiation failed: client connection to 127.2.126.61:37711: connect: Connection refused (error 111)
19:31:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:12.028842 2644 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:12.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:12.313908 2632 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } attempt: 5
19:31:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:12.317775 2632 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.126.61:37711: Network error: Client connection negotiation failed: client connection to 127.2.126.61:37711: connect: Connection refused (error 111)
19:31:12.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:12.865453 2632 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } attempt: 6
19:31:12.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:12.873090 2632 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.126.61:37711: Network error: Client connection negotiation failed: client connection to 127.2.126.61:37711: connect: Connection refused (error 111)
19:31:13.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:12.031790 2634 server_base.cc:1034] running on GCE node
19:31:13.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:12.031872 2646 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:13.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:12.029585 2643 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:13.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.316987 2634 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:31:13.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.320284 2634 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:31:13.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.321760 2634 hybrid_clock.cc:648] HybridClock initialized: now 1790623873321708 us; error 65 us; skew 500 ppm
19:31:13.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.322870 2634 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:31:13.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.333945 2634 webserver.cc:460] Webserver started at http://127.2.126.61:35791/ using document root <none> and password file <none>
19:31:13.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.334956 2634 fs_manager.cc:359] Metadata directory not provided
19:31:13.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.335139 2634 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:31:13.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.335594 2634 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:31:13.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.340421 2634 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-1/data/instance:
19:31:13.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "46f22d4108814258adb3dd866a649b87"
19:31:13.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:13 on dist-test-slave-fr74"
19:31:13.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.341560 2634 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-1/wal/instance:
19:31:13.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "46f22d4108814258adb3dd866a649b87"
19:31:13.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:13 on dist-test-slave-fr74"
19:31:13.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.349347 2634 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.003s
19:31:13.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.355618 2655 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:31:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.356812 2634 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
19:31:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.357103 2634 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-1/data,/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-1/wal
19:31:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "46f22d4108814258adb3dd866a649b87"
19:31:13.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:13 on dist-test-slave-fr74"
19:31:13.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.357427 2634 fs_report.cc:383] FS layout report
19:31:13.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:31:13.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-1/wal
19:31:13.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-1/wal
19:31:13.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-1/data/data
19:31:13.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:31:13.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:31:13.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:31:13.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:31:13.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:31:13.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:31:13.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:31:13.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:31:13.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:31:13.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:31:13.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:31:13.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.393903 2634 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:31:13.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.395315 2634 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:31:13.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.395763 2634 kserver.cc:163] Server-wide thread pool size limit: 3276
19:31:13.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.478138 2634 rpc_server.cc:307] RPC server started. Bound to: 127.2.126.61:37711
19:31:13.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.478230 2706 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.126.61:37711 every 8 connection(s)
19:31:13.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.481448 2634 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-1/data/info.pb
19:31:13.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.487140 2707 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:31:13.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.490183 2552 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu as pid 2634
19:31:13.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.490674 2552 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-1/wal/instance
19:31:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.493675 2552 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-2/wal
19:31:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-2/data
19:31:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:31:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:31:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:31:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-2/logs
19:31:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-2/data/info.pb
19:31:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:31:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:31:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:31:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:31:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:31:13.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:31:13.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:31:13.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:31:13.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.126.60:39801
19:31:13.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.126.60
19:31:13.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:31:13.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.126.20:42927
19:31:13.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:13.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:13.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:31:13.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:31:13.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.126.62:36437,127.2.126.61:37711,127.2.126.60:39801
19:31:13.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:31:13.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43327/jwks.json with env {}
19:31:13.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.503419 2707 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 } has no permanent_uuid. Determining permanent_uuid...
19:31:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.528030 2707 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } has no permanent_uuid. Determining permanent_uuid...
19:31:13.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.545682 2707 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } has no permanent_uuid. Determining permanent_uuid...
19:31:13.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:13.548225 2659 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.126.60:39801: connect: Connection refused (error 111)
19:31:13.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:13.557930 2707 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.126.60:39801: Network error: Client connection negotiation failed: client connection to 127.2.126.60:39801: connect: Connection refused (error 111)
19:31:13.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.607654 2707 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } attempt: 1
19:31:13.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:13.611549 2707 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.126.60:39801: Network error: Client connection negotiation failed: client connection to 127.2.126.60:39801: connect: Connection refused (error 111)
19:31:13.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.679926 2707 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } attempt: 2
19:31:13.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:13.684389 2707 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.126.60:39801: Network error: Client connection negotiation failed: client connection to 127.2.126.60:39801: connect: Connection refused (error 111)
19:31:13.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.775878 2707 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } attempt: 3
19:31:13.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:13.780427 2707 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.126.60:39801: Network error: Client connection negotiation failed: client connection to 127.2.126.60:39801: connect: Connection refused (error 111)
19:31:13.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:13.826547 2709 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:31:13.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:13.827087 2709 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:31:13.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:13.827759 2709 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:31:13.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:13.861191 2709 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:31:13.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:13.861650 2709 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:31:13.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:13.861861 2709 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:31:13.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:13.862200 2709 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:31:13.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:13.862553 2709 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:31:13.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:13.862771 2709 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43327/jwks.json
19:31:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.900951 2709 master_runner.cc:386] Master server non-default flags:
19:31:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.126.20:42927
19:31:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:31:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-2/data
19:31:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-2/wal
19:31:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:31:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.126.62:36437,127.2.126.61:37711,127.2.126.60:39801
19:31:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:31:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.126.60:39801
19:31:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:31:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:31:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:31:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43327/jwks.json
19:31:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-2/data/info.pb
19:31:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.126.60
19:31:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:31:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:31:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:31:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:31:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:13.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-2/logs
19:31:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:31:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:31:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:31:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:31:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:31:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:31:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:31:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:31:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:31:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.902791 2709 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:31:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.905339 2709 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:31:13.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.924247 2707 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } attempt: 4
19:31:13.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:13.932251 2707 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.126.60:39801: Network error: Client connection negotiation failed: client connection to 127.2.126.60:39801: connect: Connection refused (error 111)
19:31:13.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.933957 2632 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } attempt: 7
19:31:13.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:13.938143 2720 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:13.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:13.975536 2632 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } has no permanent_uuid. Determining permanent_uuid...
19:31:13.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:13.995880 2632 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.126.60:39801: Network error: Client connection negotiation failed: client connection to 127.2.126.60:39801: connect: Connection refused (error 111)
19:31:14.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:14.054492 2632 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } attempt: 1
19:31:14.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:14.061517 2632 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.126.60:39801: Network error: Client connection negotiation failed: client connection to 127.2.126.60:39801: connect: Connection refused (error 111)
19:31:14.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:14.143069 2632 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } attempt: 2
19:31:14.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:14.152558 2632 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.126.60:39801: Network error: Client connection negotiation failed: client connection to 127.2.126.60:39801: connect: Connection refused (error 111)
19:31:14.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:14.231899 2707 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } attempt: 5
19:31:14.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:14.239079 2632 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } attempt: 3
19:31:14.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:14.242990 2707 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.126.60:39801: Network error: Client connection negotiation failed: client connection to 127.2.126.60:39801: connect: Connection refused (error 111)
19:31:14.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:14.250754 2632 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.126.60:39801: Network error: Client connection negotiation failed: client connection to 127.2.126.60:39801: connect: Connection refused (error 111)
19:31:14.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:14.391301 2632 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } attempt: 4
19:31:14.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:14.411746 2632 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.126.60:39801: Network error: Client connection negotiation failed: client connection to 127.2.126.60:39801: connect: Connection refused (error 111)
19:31:14.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:14.695323 2632 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } attempt: 5
19:31:14.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:14.703351 2632 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.126.60:39801: Network error: Client connection negotiation failed: client connection to 127.2.126.60:39801: connect: Connection refused (error 111)
19:31:14.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:14.790920 2707 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } attempt: 6
19:31:14.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:14.818774 2707 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.126.60:39801: Network error: Client connection negotiation failed: client connection to 127.2.126.60:39801: connect: Connection refused (error 111)
19:31:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.255945 2632 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } attempt: 6
19:31:15.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:15.279270 2632 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.126.60:39801: Network error: Client connection negotiation failed: client connection to 127.2.126.60:39801: connect: Connection refused (error 111)
19:31:15.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:15.327137 2718 debug-util.cc:398] Leaking SignalData structure 0x7b0800014880 after lost signal to thread 2709
19:31:15.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:15.440313 2709 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.506s user 0.477s sys 1.016s
19:31:15.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:13.939785 2719 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:15.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:15.442268 2709 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.508s user 0.477s sys 1.017s
19:31:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:15.442634 2721 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1505 milliseconds
19:31:15.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:15.444201 2722 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:15.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.444124 2709 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:31:15.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.449131 2709 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:31:15.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.451874 2709 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:31:15.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.453305 2709 hybrid_clock.cc:648] HybridClock initialized: now 1790623875453217 us; error 67 us; skew 500 ppm
19:31:15.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.454236 2709 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:31:15.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.460907 2709 webserver.cc:460] Webserver started at http://127.2.126.60:33155/ using document root <none> and password file <none>
19:31:15.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.462108 2709 fs_manager.cc:359] Metadata directory not provided
19:31:15.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.462401 2709 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:31:15.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.462961 2709 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:31:15.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.468472 2709 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-2/data/instance:
19:31:15.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b09b48c34864cd1bbda9e53a74c0477"
19:31:15.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:15 on dist-test-slave-fr74"
19:31:15.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.469683 2709 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-2/wal/instance:
19:31:15.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b09b48c34864cd1bbda9e53a74c0477"
19:31:15.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:15 on dist-test-slave-fr74"
19:31:15.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.477471 2709 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.003s
19:31:15.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.485543 2735 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:31:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.487042 2709 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
19:31:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.487489 2709 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-2/data,/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-2/wal
19:31:15.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b09b48c34864cd1bbda9e53a74c0477"
19:31:15.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:15 on dist-test-slave-fr74"
19:31:15.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.487923 2709 fs_report.cc:383] FS layout report
19:31:15.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:31:15.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-2/wal
19:31:15.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-2/wal
19:31:15.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-2/data/data
19:31:15.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:31:15.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:31:15.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:31:15.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:31:15.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:31:15.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:31:15.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:31:15.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:31:15.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:31:15.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:31:15.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:31:15.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.538929 2709 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:31:15.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.540410 2709 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:31:15.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.541213 2709 kserver.cc:163] Server-wide thread pool size limit: 3276
19:31:15.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.629439 2709 rpc_server.cc:307] RPC server started. Bound to: 127.2.126.60:39801
19:31:15.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.629516 2786 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.126.60:39801 every 8 connection(s)
19:31:15.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.632601 2709 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-2/data/info.pb
19:31:15.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.638557 2787 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:31:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.641175 2552 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu as pid 2709
19:31:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.642118 2552 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/master-2/wal/instance
19:31:15.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.659855 2787 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 } has no permanent_uuid. Determining permanent_uuid...
19:31:15.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.692039 2787 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } has no permanent_uuid. Determining permanent_uuid...
19:31:15.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.707984 2787 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } has no permanent_uuid. Determining permanent_uuid...
19:31:15.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.728245 2787 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477: Bootstrap starting.
19:31:15.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.736739 2787 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477: Neither blocks nor log segments found. Creating new log.
19:31:15.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.739866 2787 log.cc:824] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477: Log is configured to *not* fsync() on all Append() calls
19:31:15.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.746958 2787 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477: No bootstrap required, opened a new log
19:31:15.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.772032 2787 raft_consensus.cc:348] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 } } peers { permanent_uuid: "46f22d4108814258adb3dd866a649b87" member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } } peers { permanent_uuid: "8b09b48c34864cd1bbda9e53a74c0477" member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } }
19:31:15.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.772645 2787 raft_consensus.cc:374] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:31:15.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.772841 2787 raft_consensus.cc:729] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8b09b48c34864cd1bbda9e53a74c0477, State: Initialized, Role: FOLLOWER
19:31:15.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.773677 2787 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 } } peers { permanent_uuid: "46f22d4108814258adb3dd866a649b87" member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } } peers { permanent_uuid: "8b09b48c34864cd1bbda9e53a74c0477" member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } }
19:31:15.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.777366 2796 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 } } peers { permanent_uuid: "46f22d4108814258adb3dd866a649b87" member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } } peers { permanent_uuid: "8b09b48c34864cd1bbda9e53a74c0477" member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } } }
19:31:15.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.778249 2796 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [sys.catalog]: This master's current role is: FOLLOWER
19:31:15.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.779122 2787 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [sys.catalog]: configured and running, proceeding with master startup.
19:31:15.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:15.796525 2741 tablet.cc:2369] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:31:15.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:15.809926 2807 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:15.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:15.810704 2807 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:15.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.804462 2709 master_runner.cc:186] Error getting master registration for 127.2.126.62:36437: OK, instance_id { permanent_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" instance_seqno: 1790623871552323 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
19:31:15.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.835235 2709 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:31:15.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.879406 2707 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } attempt: 7
19:31:15.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.903005 2707 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87: Bootstrap starting.
19:31:15.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.914085 2707 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87: Neither blocks nor log segments found. Creating new log.
19:31:15.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.916985 2707 log.cc:824] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87: Log is configured to *not* fsync() on all Append() calls
19:31:15.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.925422 2707 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87: No bootstrap required, opened a new log
19:31:15.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.939738 2796 raft_consensus.cc:482] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:31:15.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.940276 2796 raft_consensus.cc:504] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 } } peers { permanent_uuid: "46f22d4108814258adb3dd866a649b87" member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } } peers { permanent_uuid: "8b09b48c34864cd1bbda9e53a74c0477" member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } }
19:31:15.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.943727 2796 leader_election.cc:288] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e5f9ceb87f6146dd8c7532c66ed0de44 (127.2.126.62:36437), 46f22d4108814258adb3dd866a649b87 (127.2.126.61:37711)
19:31:15.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.963869 2707 raft_consensus.cc:348] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 } } peers { permanent_uuid: "46f22d4108814258adb3dd866a649b87" member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } } peers { permanent_uuid: "8b09b48c34864cd1bbda9e53a74c0477" member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } }
19:31:15.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.965483 2707 raft_consensus.cc:374] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:31:15.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.966050 2707 raft_consensus.cc:729] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 46f22d4108814258adb3dd866a649b87, State: Initialized, Role: FOLLOWER
19:31:15.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.967227 2707 consensus_queue.cc:260] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 } } peers { permanent_uuid: "46f22d4108814258adb3dd866a649b87" member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } } peers { permanent_uuid: "8b09b48c34864cd1bbda9e53a74c0477" member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } }
19:31:15.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.972018 2811 sys_catalog.cc:455] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 } } peers { permanent_uuid: "46f22d4108814258adb3dd866a649b87" member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } } peers { permanent_uuid: "8b09b48c34864cd1bbda9e53a74c0477" member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } } }
19:31:15.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.973727 2707 sys_catalog.cc:564] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87 [sys.catalog]: configured and running, proceeding with master startup.
19:31:15.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.973650 2811 sys_catalog.cc:458] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87 [sys.catalog]: This master's current role is: FOLLOWER
19:31:15.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.974004 2682 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8b09b48c34864cd1bbda9e53a74c0477" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "46f22d4108814258adb3dd866a649b87" is_pre_election: true
19:31:15.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.974867 2682 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8b09b48c34864cd1bbda9e53a74c0477 in term 0.
19:31:15.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.976827 2736 leader_election.cc:302] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 46f22d4108814258adb3dd866a649b87, 8b09b48c34864cd1bbda9e53a74c0477; no voters:
19:31:15.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.977806 2796 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [term 0 FOLLOWER]: Leader pre-election won for term 1
19:31:15.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.978590 2796 raft_consensus.cc:482] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
19:31:15.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.979225 2796 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [term 0 FOLLOWER]: Advancing to term 1
19:31:15.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.990824 2796 raft_consensus.cc:504] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 } } peers { permanent_uuid: "46f22d4108814258adb3dd866a649b87" member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } } peers { permanent_uuid: "8b09b48c34864cd1bbda9e53a74c0477" member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } }
19:31:15.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.998870 2682 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8b09b48c34864cd1bbda9e53a74c0477" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "46f22d4108814258adb3dd866a649b87"
19:31:16.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:15.999661 2682 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87 [term 0 FOLLOWER]: Advancing to term 1
19:31:16.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.013549 2796 leader_election.cc:288] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [CANDIDATE]: Term 1 election: Requested vote from peers e5f9ceb87f6146dd8c7532c66ed0de44 (127.2.126.62:36437), 46f22d4108814258adb3dd866a649b87 (127.2.126.61:37711)
19:31:16.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.022905 2682 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8b09b48c34864cd1bbda9e53a74c0477 in term 1.
19:31:16.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.024120 2736 leader_election.cc:302] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 46f22d4108814258adb3dd866a649b87, 8b09b48c34864cd1bbda9e53a74c0477; no voters:
19:31:16.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.025014 2796 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [term 1 FOLLOWER]: Leader election won for term 1
19:31:16.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.039391 2796 raft_consensus.cc:686] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [term 1 LEADER]: Becoming Leader. State: Replica: 8b09b48c34864cd1bbda9e53a74c0477, State: Running, Role: LEADER
19:31:16.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.040767 2796 consensus_queue.cc:237] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 } } peers { permanent_uuid: "46f22d4108814258adb3dd866a649b87" member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } } peers { permanent_uuid: "8b09b48c34864cd1bbda9e53a74c0477" member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } }
19:31:16.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.053380 2822 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8b09b48c34864cd1bbda9e53a74c0477. Latest consensus state: current_term: 1 leader_uuid: "8b09b48c34864cd1bbda9e53a74c0477" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 } } peers { permanent_uuid: "46f22d4108814258adb3dd866a649b87" member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } } peers { permanent_uuid: "8b09b48c34864cd1bbda9e53a74c0477" member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } } }
19:31:16.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.054824 2822 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [sys.catalog]: This master's current role is: LEADER
19:31:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.058233 2825 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
19:31:16.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.065833 2825 catalog_manager.cc:1484] Initializing Kudu cluster ID...
19:31:16.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:16.079885 2823 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:16.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:16.080257 2823 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:16.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.109544 2682 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87 [term 1 FOLLOWER]: Refusing update from remote peer 8b09b48c34864cd1bbda9e53a74c0477: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
19:31:16.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.111321 2796 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [LEADER]: Connected to new peer: Peer: permanent_uuid: "46f22d4108814258adb3dd866a649b87" member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
19:31:16.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.042341 2634 master_runner.cc:186] Error getting master registration for 127.2.126.62:36437: OK, instance_id { permanent_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" instance_seqno: 1790623871552323 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
19:31:16.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.118541 2634 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:31:16.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.148895 2811 sys_catalog.cc:455] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8b09b48c34864cd1bbda9e53a74c0477. Latest consensus state: current_term: 1 leader_uuid: "8b09b48c34864cd1bbda9e53a74c0477" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 } } peers { permanent_uuid: "46f22d4108814258adb3dd866a649b87" member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } } peers { permanent_uuid: "8b09b48c34864cd1bbda9e53a74c0477" member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } } }
19:31:16.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.151108 2811 sys_catalog.cc:458] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87 [sys.catalog]: This master's current role is: FOLLOWER
19:31:16.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:16.155685 2661 tablet.cc:2369] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:31:16.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.156849 2827 mvcc.cc:205] Tried to move back new op lower bound from 7334395396515385344 to 7334395396275920896. Current Snapshot: MvccSnapshot[applied={T|T < 7334395396515385344}]
19:31:16.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.167081 2811 sys_catalog.cc:455] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "8b09b48c34864cd1bbda9e53a74c0477" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 } } peers { permanent_uuid: "46f22d4108814258adb3dd866a649b87" member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } } peers { permanent_uuid: "8b09b48c34864cd1bbda9e53a74c0477" member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } } }
19:31:16.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.168573 2811 sys_catalog.cc:458] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87 [sys.catalog]: This master's current role is: FOLLOWER
19:31:16.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.176512 2796 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "8b09b48c34864cd1bbda9e53a74c0477" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 } } peers { permanent_uuid: "46f22d4108814258adb3dd866a649b87" member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } } peers { permanent_uuid: "8b09b48c34864cd1bbda9e53a74c0477" member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } } }
19:31:16.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.177871 2796 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [sys.catalog]: This master's current role is: LEADER
19:31:16.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.190151 2825 catalog_manager.cc:1347] Generated new cluster ID: 357b8a9c47534450a9652a79067100ac
19:31:16.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.190469 2825 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
19:31:16.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.247605 2825 catalog_manager.cc:1370] Generated new certificate authority record
19:31:16.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.250897 2825 catalog_manager.cc:1504] Loading token signing keys...
19:31:16.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.281776 2825 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477: Generated new TSK 0
19:31:16.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.284713 2825 catalog_manager.cc:1514] Initializing in-progress tserver states...
19:31:16.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.303838 2552 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:16.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:16.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-0/wal
19:31:16.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-0/data
19:31:16.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:31:16.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:31:16.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:31:16.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:16.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:16.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:16.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-0/logs
19:31:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-0/data/info.pb
19:31:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:31:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:31:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:31:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:31:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:31:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:31:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.126.1:0
19:31:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.126.1
19:31:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.126.1
19:31:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:31:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.126.62:36437,127.2.126.61:37711,127.2.126.60:39801
19:31:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.126.20:42927
19:31:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:31:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.312780 2632 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } attempt: 7
19:31:16.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.368569 2632 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44: Bootstrap starting.
19:31:16.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.392467 2632 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44: Neither blocks nor log segments found. Creating new log.
19:31:16.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.399870 2632 log.cc:824] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44: Log is configured to *not* fsync() on all Append() calls
19:31:16.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.425379 2632 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44: No bootstrap required, opened a new log
19:31:16.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.529501 2632 raft_consensus.cc:348] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 } } peers { permanent_uuid: "46f22d4108814258adb3dd866a649b87" member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } } peers { permanent_uuid: "8b09b48c34864cd1bbda9e53a74c0477" member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } }
19:31:16.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.533151 2632 raft_consensus.cc:374] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:31:16.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.533859 2632 raft_consensus.cc:729] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e5f9ceb87f6146dd8c7532c66ed0de44, State: Initialized, Role: FOLLOWER
19:31:16.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.536403 2632 consensus_queue.cc:260] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 } } peers { permanent_uuid: "46f22d4108814258adb3dd866a649b87" member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } } peers { permanent_uuid: "8b09b48c34864cd1bbda9e53a74c0477" member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } }
19:31:16.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.555827 2836 sys_catalog.cc:455] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 } } peers { permanent_uuid: "46f22d4108814258adb3dd866a649b87" member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } } peers { permanent_uuid: "8b09b48c34864cd1bbda9e53a74c0477" member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } } }
19:31:16.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.560879 2837 raft_consensus.cc:482] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:31:16.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.560482 2836 sys_catalog.cc:458] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44 [sys.catalog]: This master's current role is: FOLLOWER
19:31:16.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.562517 2837 raft_consensus.cc:504] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 } } peers { permanent_uuid: "46f22d4108814258adb3dd866a649b87" member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } } peers { permanent_uuid: "8b09b48c34864cd1bbda9e53a74c0477" member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } }
19:31:16.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.573082 2632 sys_catalog.cc:564] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44 [sys.catalog]: configured and running, proceeding with master startup.
19:31:16.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.575618 2607 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8b09b48c34864cd1bbda9e53a74c0477" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" is_pre_election: true
19:31:16.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.576861 2606 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8b09b48c34864cd1bbda9e53a74c0477" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44"
19:31:16.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.587241 2607 raft_consensus.cc:2454] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8b09b48c34864cd1bbda9e53a74c0477 in term 0.
19:31:16.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.595512 2605 raft_consensus.cc:3037] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44 [term 0 FOLLOWER]: Advancing to term 1
19:31:16.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.608664 2605 raft_consensus.cc:1261] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44 [term 1 FOLLOWER]: Refusing update from remote peer 8b09b48c34864cd1bbda9e53a74c0477: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
19:31:16.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.611941 2822 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
19:31:16.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.624760 2681 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "46f22d4108814258adb3dd866a649b87" is_pre_election: true
19:31:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.637948 2837 leader_election.cc:288] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 46f22d4108814258adb3dd866a649b87 (127.2.126.61:37711), 8b09b48c34864cd1bbda9e53a74c0477 (127.2.126.60:39801)
19:31:16.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.639734 2762 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8b09b48c34864cd1bbda9e53a74c0477" is_pre_election: true
19:31:16.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.641834 2584 leader_election.cc:302] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e5f9ceb87f6146dd8c7532c66ed0de44; no voters: 46f22d4108814258adb3dd866a649b87, 8b09b48c34864cd1bbda9e53a74c0477
19:31:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.683771 2837 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:31:16.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.688920 2836 sys_catalog.cc:455] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8b09b48c34864cd1bbda9e53a74c0477. Latest consensus state: current_term: 1 leader_uuid: "8b09b48c34864cd1bbda9e53a74c0477" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 } } peers { permanent_uuid: "46f22d4108814258adb3dd866a649b87" member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } } peers { permanent_uuid: "8b09b48c34864cd1bbda9e53a74c0477" member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } } }
19:31:16.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.688980 2843 sys_catalog.cc:455] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "8b09b48c34864cd1bbda9e53a74c0477" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 } } peers { permanent_uuid: "46f22d4108814258adb3dd866a649b87" member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } } peers { permanent_uuid: "8b09b48c34864cd1bbda9e53a74c0477" member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } } }
19:31:16.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.689762 2836 sys_catalog.cc:458] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44 [sys.catalog]: This master's current role is: FOLLOWER
19:31:16.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.689810 2843 sys_catalog.cc:458] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44 [sys.catalog]: This master's current role is: FOLLOWER
19:31:16.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:16.750068 2853 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
19:31:17.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:16.997110 2834 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:31:17.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:16.998510 2834 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:31:17.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:16.999419 2834 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:31:17.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:17.042665 2834 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:31:17.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:17.043831 2834 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.126.1
19:31:17.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:17.086146 2823 catalog_manager.cc:1259] Loaded cluster ID: 357b8a9c47534450a9652a79067100ac
19:31:17.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:17.086755 2823 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87: loading cluster ID for follower catalog manager: success
19:31:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:17.090284 2834 tablet_server_runner.cc:78] Tablet server non-default flags:
19:31:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.126.20:42927
19:31:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:31:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-0/data
19:31:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-0/wal
19:31:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.126.1:0
19:31:17.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:31:17.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:17.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:17.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-0/data/info.pb
19:31:17.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.126.1
19:31:17.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:31:17.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.126.62:36437,127.2.126.61:37711,127.2.126.60:39801
19:31:17.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:31:17.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:17.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:31:17.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:31:17.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:17.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.126.1
19:31:17.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-0/logs
19:31:17.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:17.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:31:17.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:31:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:31:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:31:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:31:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:31:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:31:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:31:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:31:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:17.092089 2834 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:31:17.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:17.094053 2834 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:31:17.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:17.107632 2823 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87: acquiring CA information for follower catalog manager: success
19:31:17.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:17.115248 2823 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 46f22d4108814258adb3dd866a649b87: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
19:31:17.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:17.130560 2859 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:17.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:17.200883 2822 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "8b09b48c34864cd1bbda9e53a74c0477" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e5f9ceb87f6146dd8c7532c66ed0de44" member_type: VOTER last_known_addr { host: "127.2.126.62" port: 36437 } } peers { permanent_uuid: "46f22d4108814258adb3dd866a649b87" member_type: VOTER last_known_addr { host: "127.2.126.61" port: 37711 } } peers { permanent_uuid: "8b09b48c34864cd1bbda9e53a74c0477" member_type: VOTER last_known_addr { host: "127.2.126.60" port: 39801 } } }
19:31:17.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:17.202140 2822 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b09b48c34864cd1bbda9e53a74c0477 [sys.catalog]: This master's current role is: LEADER
19:31:17.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:17.792429 2853 catalog_manager.cc:1259] Loaded cluster ID: 357b8a9c47534450a9652a79067100ac
19:31:17.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:17.793149 2853 catalog_manager.cc:1552] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44: loading cluster ID for follower catalog manager: success
19:31:17.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:17.832660 2853 catalog_manager.cc:1574] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44: acquiring CA information for follower catalog manager: success
19:31:17.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:17.859839 2853 catalog_manager.cc:1602] T 00000000000000000000000000000000 P e5f9ceb87f6146dd8c7532c66ed0de44: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
19:31:19.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:17.139703 2860 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:19.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:18.525833 2858 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 2834
19:31:19.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:19.930452 2858 kernel_stack_watchdog.cc:198] Thread 2834 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 384ms:
19:31:19.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
19:31:19.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
19:31:19.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
19:31:19.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
19:31:19.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:19.935755 2834 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.797s user 0.000s sys 0.004s
19:31:19.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:19.936116 2834 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.797s user 0.000s sys 0.004s
19:31:19.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:19.936301 2861 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2796 milliseconds
19:31:19.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:19.942708 2834 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:31:19.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:19.943661 2862 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:19.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:19.945726 2834 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:31:19.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:19.955955 2834 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:31:19.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:19.959918 2834 hybrid_clock.cc:648] HybridClock initialized: now 1790623879959842 us; error 82 us; skew 500 ppm
19:31:19.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:19.961189 2834 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:31:20.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.006340 2834 webserver.cc:460] Webserver started at http://127.2.126.1:33173/ using document root <none> and password file <none>
19:31:20.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.010598 2834 fs_manager.cc:359] Metadata directory not provided
19:31:20.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.011876 2834 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:31:20.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.013111 2834 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:31:20.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.021857 2834 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-0/data/instance:
19:31:20.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4342abce2d684aa186952a7a4e28646f"
19:31:20.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:20 on dist-test-slave-fr74"
19:31:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.023809 2834 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-0/wal/instance:
19:31:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4342abce2d684aa186952a7a4e28646f"
19:31:20.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:20 on dist-test-slave-fr74"
19:31:20.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.040813 2834 fs_manager.cc:691] Time spent creating directory manager: real 0.016s user 0.013s sys 0.000s
19:31:20.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.054922 2874 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:31:20.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.056633 2834 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.010s sys 0.000s
19:31:20.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.057179 2834 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-0/data,/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-0/wal
19:31:20.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4342abce2d684aa186952a7a4e28646f"
19:31:20.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:20 on dist-test-slave-fr74"
19:31:20.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.057863 2834 fs_report.cc:383] FS layout report
19:31:20.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:31:20.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-0/wal
19:31:20.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-0/wal
19:31:20.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-0/data/data
19:31:20.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:31:20.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:31:20.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:31:20.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:31:20.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:31:20.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:31:20.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:31:20.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:31:20.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:31:20.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:31:20.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:31:20.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.148729 2834 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:31:20.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.151371 2834 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:31:20.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.152124 2834 kserver.cc:163] Server-wide thread pool size limit: 3276
19:31:20.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.162035 2834 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
19:31:20.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.177345 2834 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:31:20.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.178119 2834 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.001s user 0.000s sys 0.000s
19:31:20.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.178615 2834 ts_tablet_manager.cc:610] Registered 0 tablets
19:31:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.178982 2834 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
19:31:20.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.787092 2834 rpc_server.cc:307] RPC server started. Bound to: 127.2.126.1:43769
19:31:20.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.787231 2986 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.126.1:43769 every 8 connection(s)
19:31:20.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.790729 2834 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-0/data/info.pb
19:31:20.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.801303 2552 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu as pid 2834
19:31:20.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.801946 2552 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-0/wal/instance
19:31:20.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.826206 2552 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:20.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:20.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-1/wal
19:31:20.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-1/data
19:31:20.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:31:20.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:31:20.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-1/logs
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-1/data/info.pb
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.126.2:0
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.126.2
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.126.2
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.126.62:36437,127.2.126.61:37711,127.2.126.60:39801
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.126.20:42927
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:20.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:31:20.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.918366 2987 heartbeater.cc:347] Connected to a master server at 127.2.126.60:39801
19:31:20.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.918843 2987 heartbeater.cc:464] Registering TS with master...
19:31:20.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.919982 2987 heartbeater.cc:511] Master 127.2.126.60:39801 requested a full tablet report, sending...
19:31:20.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.925061 2752 ts_manager.cc:194] Registered new tserver with Master: 4342abce2d684aa186952a7a4e28646f (127.2.126.1:43769)
19:31:20.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.928120 2752 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.2.126.1:48673
19:31:20.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.935770 2992 heartbeater.cc:347] Connected to a master server at 127.2.126.62:36437
19:31:20.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.936245 2992 heartbeater.cc:464] Registering TS with master...
19:31:20.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.940071 2992 heartbeater.cc:511] Master 127.2.126.62:36437 requested a full tablet report, sending...
19:31:20.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.943526 2597 ts_manager.cc:194] Registered new tserver with Master: 4342abce2d684aa186952a7a4e28646f (127.2.126.1:43769)
19:31:20.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.954456 2988 heartbeater.cc:347] Connected to a master server at 127.2.126.61:37711
19:31:20.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.954798 2988 heartbeater.cc:464] Registering TS with master...
19:31:20.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.955830 2988 heartbeater.cc:511] Master 127.2.126.61:37711 requested a full tablet report, sending...
19:31:20.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:20.958729 2672 ts_manager.cc:194] Registered new tserver with Master: 4342abce2d684aa186952a7a4e28646f (127.2.126.1:43769)
19:31:21.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:21.273533 2991 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:31:21.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:21.274120 2991 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:31:21.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:21.274785 2991 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:31:21.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:21.310170 2991 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:31:21.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:21.311154 2991 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.126.2
19:31:21.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:21.346869 2991 tablet_server_runner.cc:78] Tablet server non-default flags:
19:31:21.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:21.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.126.20:42927
19:31:21.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:21.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:31:21.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-1/data
19:31:21.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-1/wal
19:31:21.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:21.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:21.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.126.2:0
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-1/data/info.pb
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.126.2
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.126.62:36437,127.2.126.61:37711,127.2.126.60:39801
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.126.2
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-1/logs
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:31:21.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:21.348484 2991 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:31:21.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:21.350673 2991 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:31:21.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:21.375830 3004 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:21.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:21.941640 2987 heartbeater.cc:503] Master 127.2.126.60:39801 was elected leader, sending a full tablet report...
19:31:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:21.375825 3003 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:21.378571 3006 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:22.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:22.693001 3005 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
19:31:22.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.693109 2991 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:31:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.699406 2991 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:31:22.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.702606 2991 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:31:22.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.703979 2991 hybrid_clock.cc:648] HybridClock initialized: now 1790623882703952 us; error 68 us; skew 500 ppm
19:31:22.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.704921 2991 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:31:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.711598 2991 webserver.cc:460] Webserver started at http://127.2.126.2:39085/ using document root <none> and password file <none>
19:31:22.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.712488 2991 fs_manager.cc:359] Metadata directory not provided
19:31:22.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.712675 2991 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:31:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.713078 2991 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:31:22.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.717893 2991 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-1/data/instance:
19:31:22.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a579a5d43fe7416b825c07e7c3cdcbad"
19:31:22.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:22 on dist-test-slave-fr74"
19:31:22.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.718978 2991 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-1/wal/instance:
19:31:22.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a579a5d43fe7416b825c07e7c3cdcbad"
19:31:22.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:22 on dist-test-slave-fr74"
19:31:22.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.725850 2991 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
19:31:22.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.731237 3014 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:31:22.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.732700 2991 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
19:31:22.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.733067 2991 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-1/data,/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-1/wal
19:31:22.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a579a5d43fe7416b825c07e7c3cdcbad"
19:31:22.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:22 on dist-test-slave-fr74"
19:31:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.733477 2991 fs_report.cc:383] FS layout report
19:31:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:31:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-1/wal
19:31:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-1/wal
19:31:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-1/data/data
19:31:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:31:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:31:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:31:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:31:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:31:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:31:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:31:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:31:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:31:22.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:31:22.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:31:22.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.784920 2991 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:31:22.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.786444 2991 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:31:22.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.786999 2991 kserver.cc:163] Server-wide thread pool size limit: 3276
19:31:22.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.790452 2991 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
19:31:22.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.797133 2991 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:31:22.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.797438 2991 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
19:31:22.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.797729 2991 ts_tablet_manager.cc:610] Registered 0 tablets
19:31:22.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.797955 2991 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
19:31:22.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.979049 2991 rpc_server.cc:307] RPC server started. Bound to: 127.2.126.2:36545
19:31:22.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.979286 3126 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.126.2:36545 every 8 connection(s)
19:31:22.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.981782 2991 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-1/data/info.pb
19:31:22.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.987814 2552 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu as pid 2991
19:31:22.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:22.988312 2552 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-1/wal/instance
19:31:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:23.009977 2552 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-2/wal
19:31:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-2/data
19:31:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:31:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:31:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:31:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-2/logs
19:31:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-2/data/info.pb
19:31:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:31:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:31:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:31:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:31:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:31:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:31:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.126.3:0
19:31:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.126.3
19:31:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.126.3
19:31:23.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:31:23.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.126.62:36437,127.2.126.61:37711,127.2.126.60:39801
19:31:23.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.126.20:42927
19:31:23.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:23.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:23.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:31:23.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:23.039538 3129 heartbeater.cc:347] Connected to a master server at 127.2.126.62:36437
19:31:23.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:23.040042 3129 heartbeater.cc:464] Registering TS with master...
19:31:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:23.041342 3129 heartbeater.cc:511] Master 127.2.126.62:36437 requested a full tablet report, sending...
19:31:23.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:23.041867 3127 heartbeater.cc:347] Connected to a master server at 127.2.126.60:39801
19:31:23.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:23.041946 3128 heartbeater.cc:347] Connected to a master server at 127.2.126.61:37711
19:31:23.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:23.042219 3127 heartbeater.cc:464] Registering TS with master...
19:31:23.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:23.042265 3128 heartbeater.cc:464] Registering TS with master...
19:31:23.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:23.043430 3128 heartbeater.cc:511] Master 127.2.126.61:37711 requested a full tablet report, sending...
19:31:23.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:23.043478 3127 heartbeater.cc:511] Master 127.2.126.60:39801 requested a full tablet report, sending...
19:31:23.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:23.045063 2597 ts_manager.cc:194] Registered new tserver with Master: a579a5d43fe7416b825c07e7c3cdcbad (127.2.126.2:36545)
19:31:23.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:23.046597 2672 ts_manager.cc:194] Registered new tserver with Master: a579a5d43fe7416b825c07e7c3cdcbad (127.2.126.2:36545)
19:31:23.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:23.047650 2752 ts_manager.cc:194] Registered new tserver with Master: a579a5d43fe7416b825c07e7c3cdcbad (127.2.126.2:36545)
19:31:23.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:23.049095 2752 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.2.126.2:59571
19:31:23.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:23.384840 3137 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:31:23.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:23.385352 3137 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:31:23.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:23.385905 3137 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:31:23.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:23.422130 3137 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:31:23.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:23.422948 3137 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.126.3
19:31:23.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:23.459906 3137 tablet_server_runner.cc:78] Tablet server non-default flags:
19:31:23.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:23.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.126.20:42927
19:31:23.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:23.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:31:23.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-2/data
19:31:23.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-2/wal
19:31:23.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.126.3:0
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-2/data/info.pb
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.126.3
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.126.62:36437,127.2.126.61:37711,127.2.126.60:39801
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.126.3
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-2/logs
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:31:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:23.461092 3137 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:31:23.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:23.462946 3137 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:31:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:23.485980 3146 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:24.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.053892 3127 heartbeater.cc:503] Master 127.2.126.60:39801 was elected leader, sending a full tablet report...
19:31:24.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:23.487953 3143 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:24.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:23.486001 3144 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:24.813774 3145 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1325 milliseconds
19:31:24.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.813946 3137 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:31:24.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.815706 3137 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:31:24.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.818877 3137 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:31:24.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.820362 3137 hybrid_clock.cc:648] HybridClock initialized: now 1790623884820323 us; error 50 us; skew 500 ppm
19:31:24.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.821182 3137 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:31:24.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.827430 3137 webserver.cc:460] Webserver started at http://127.2.126.3:37307/ using document root <none> and password file <none>
19:31:24.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.828307 3137 fs_manager.cc:359] Metadata directory not provided
19:31:24.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.828501 3137 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:31:24.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.828876 3137 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:31:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.834934 3137 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-2/data/instance:
19:31:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9ce6d657e33a4ca6b5f1950c54278565"
19:31:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:24 on dist-test-slave-fr74"
19:31:24.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.836225 3137 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-2/wal/instance:
19:31:24.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9ce6d657e33a4ca6b5f1950c54278565"
19:31:24.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:24 on dist-test-slave-fr74"
19:31:24.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.843829 3137 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
19:31:24.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.849543 3155 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:31:24.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.851094 3137 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
19:31:24.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.851537 3137 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-2/data,/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-2/wal
19:31:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9ce6d657e33a4ca6b5f1950c54278565"
19:31:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:24 on dist-test-slave-fr74"
19:31:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.851886 3137 fs_report.cc:383] FS layout report
19:31:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:31:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-2/wal
19:31:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-2/wal
19:31:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-2/data/data
19:31:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:31:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:31:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:31:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:31:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:31:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:31:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:31:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:31:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:31:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:31:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:31:24.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.897329 3137 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:31:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.899050 3137 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:31:24.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.899598 3137 kserver.cc:163] Server-wide thread pool size limit: 3276
19:31:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.902271 3137 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
19:31:24.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.906863 3137 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:31:24.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.907069 3137 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
19:31:24.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.907269 3137 ts_tablet_manager.cc:610] Registered 0 tablets
19:31:24.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:24.907418 3137 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
19:31:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:25.088711 3137 rpc_server.cc:307] RPC server started. Bound to: 127.2.126.3:41321
19:31:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:25.088855 3267 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.126.3:41321 every 8 connection(s)
19:31:25.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:25.091532 3137 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-2/data/info.pb
19:31:25.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:25.098371 2552 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu as pid 3137
19:31:25.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:25.098841 2552 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster3587082233689753606/ts-2/wal/instance
19:31:25.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:25.150240 3268 heartbeater.cc:347] Connected to a master server at 127.2.126.60:39801
19:31:25.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:25.150630 3269 heartbeater.cc:347] Connected to a master server at 127.2.126.61:37711
19:31:25.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:25.150708 3268 heartbeater.cc:464] Registering TS with master...
19:31:25.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:25.150933 3269 heartbeater.cc:464] Registering TS with master...
19:31:25.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:25.152088 3269 heartbeater.cc:511] Master 127.2.126.61:37711 requested a full tablet report, sending...
19:31:25.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:25.152137 3268 heartbeater.cc:511] Master 127.2.126.60:39801 requested a full tablet report, sending...
19:31:25.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:25.152493 3270 heartbeater.cc:347] Connected to a master server at 127.2.126.62:36437
19:31:25.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:25.153093 3270 heartbeater.cc:464] Registering TS with master...
19:31:25.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:25.154964 3270 heartbeater.cc:511] Master 127.2.126.62:36437 requested a full tablet report, sending...
19:31:25.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:25.155223 2752 ts_manager.cc:194] Registered new tserver with Master: 9ce6d657e33a4ca6b5f1950c54278565 (127.2.126.3:41321)
19:31:25.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:25.157537 2752 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.2.126.3:56429
19:31:25.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:25.158028 2597 ts_manager.cc:194] Registered new tserver with Master: 9ce6d657e33a4ca6b5f1950c54278565 (127.2.126.3:41321)
19:31:25.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:25.159610 2672 ts_manager.cc:194] Registered new tserver with Master: 9ce6d657e33a4ca6b5f1950c54278565 (127.2.126.3:41321)
19:31:25.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:25.169651 2552 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
19:31:25.181 [DEBUG - main] (MiniKuduCluster.java:195) Response:
19:31:25.183 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
19:31:25.194 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.2.126.62"
port: 36437
}
bound_http_address {
host: "127.2.126.62"
port: 39111
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.2.126.61"
port: 37711
}
bound_http_address {
host: "127.2.126.61"
port: 35791
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.2.126.60"
port: 39801
}
bound_http_address {
host: "127.2.126.60"
port: 33155
}
}
}
19:31:25.200 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
19:31:25.203 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.2.126.1"
port: 43769
}
bound_http_address {
host: "127.2.126.1"
port: 33173
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.2.126.2"
port: 36545
}
bound_http_address {
host: "127.2.126.2"
port: 39085
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.2.126.3"
port: 41321
}
bound_http_address {
host: "127.2.126.3"
port: 37307
}
}
}
19:31:25.203 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
19:31:25.423 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
19:31:25.547 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskDtkeQs/test-tmp
19:31:25.548 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
19:31:25.549 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskDtkeQs/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
19:31:25.552 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
19:31:25.940 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:31:25.941 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
19:31:25.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:25.944173 3279 mini_oidc.cc:152] Starting JWKS server
19:31:25.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:25.948086 3279 webserver.cc:460] Webserver started at http://127.0.0.1:43921/ using document root <none> and password file <none>
19:31:25.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:25.948585 3279 mini_oidc.cc:189] Starting OIDC Discovery server
19:31:25.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:25.950666 3279 webserver.cc:460] Webserver started at http://127.0.0.1:38567/ using document root <none> and password file <none>
19:31:26.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
19:31:26.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/principal' for realm 'KRBTEST.COM',
19:31:26.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
19:31:26.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:26 dist-test-slave-fr74 krb5kdc[3292](info): setting up network...
19:31:26.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
19:31:26.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:26 dist-test-slave-fr74 krb5kdc[3292](info): set up 2 sockets
19:31:26.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:26 dist-test-slave-fr74 krb5kdc[3292](info): commencing operation
19:31:26.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
19:31:26.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:26.162732 3268 heartbeater.cc:503] Master 127.2.126.60:39801 was elected leader, sending a full tablet report...
19:31:27.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:27.134245 3279 mini_kdc.cc:121] Time spent starting KDC: real 1.154s user 0.003s sys 0.023s
19:31:27.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
19:31:27.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:31:27.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
19:31:27.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
19:31:27.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:31:27.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
19:31:27.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
19:31:27.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:31:27.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
19:31:27.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:31:27.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/test-user.keytab.
19:31:27.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/test-user.keytab.
19:31:27.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:27 dist-test-slave-fr74 krb5kdc[3292](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790623887, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:31:27.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
19:31:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T19:31:27Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
19:31:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T19:31:27Z Disabled control of system clock
19:31:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.3.51.254@KRBTEST.COM; defaulting to no policy
19:31:27.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:31:27.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.3.51.254@KRBTEST.COM" created.
19:31:27.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:31:27.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.51.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kudu_127.3.51.254.keytab.
19:31:27.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.51.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kudu_127.3.51.254.keytab.
19:31:27.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:27.338148 3279 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:27.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:27.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/wal
19:31:27.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/data
19:31:27.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:31:27.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:31:27.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:31:27.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:27.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:27.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:27.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/logs
19:31:27.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/data/info.pb
19:31:27.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.51.254:41867
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.51.254
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kudu_127.3.51.254.keytab
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.51.254
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.51.212:37303
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.51.254:41867,127.3.51.253:45857,127.3.51.252:44579
19:31:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:31:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43921/jwks.json
19:31:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:31:27.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:27.688686 3306 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:31:27.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:27.689245 3306 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:31:27.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:27.689977 3306 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:31:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:27.722157 3306 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:31:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:27.722457 3306 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:31:27.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:27.722849 3306 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:31:27.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:27.723171 3306 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:31:27.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:27.723500 3306 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:31:27.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:27.723825 3306 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:31:27.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:27.724059 3306 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43921/jwks.json
19:31:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:27.765003 3306 master_runner.cc:386] Master server non-default flags:
19:31:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.51.212:37303
19:31:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:31:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/data
19:31:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/wal
19:31:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:31:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:31:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.51.254:41867,127.3.51.253:45857,127.3.51.252:44579
19:31:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kudu_127.3.51.254.keytab
19:31:27.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.51.254
19:31:27.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:27.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:27.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:31:27.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.51.254:41867
19:31:27.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43921/jwks.json
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/data/info.pb
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.51.254
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/logs
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:31:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:31:27.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:31:27.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:27.766502 3306 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:31:27.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:27.767829 3306 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:31:27.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:27.805639 3312 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:29.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:27.806543 3315 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:29.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:27.807752 3306 server_base.cc:1034] running on GCE node
19:31:29.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:27.809857 3313 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.092901 3306 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:31:29.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.096060 3306 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:31:29.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.097880 3306 hybrid_clock.cc:648] HybridClock initialized: now 1790623889097786 us; error 173 us; skew 500 ppm
19:31:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.099431 3306 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:31:29.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:29 dist-test-slave-fr74 krb5kdc[3292](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790623889, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.51.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:31:29.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.106001 3306 init.cc:377] Logged in from keytab as kudu/127.3.51.254@KRBTEST.COM (short username kudu)
19:31:29.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.111708 3306 webserver.cc:460] Webserver started at http://127.3.51.254:42115/ using document root <none> and password file <none>
19:31:29.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.112587 3306 fs_manager.cc:359] Metadata directory not provided
19:31:29.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.112767 3306 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:31:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.113175 3306 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:31:29.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.118553 3306 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/data/instance:
19:31:29.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82752565aafb44ceb7ba8b34162177c9"
19:31:29.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:29 on dist-test-slave-fr74"
19:31:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.120110 3306 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/wal/instance:
19:31:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82752565aafb44ceb7ba8b34162177c9"
19:31:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:29 on dist-test-slave-fr74"
19:31:29.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.128369 3306 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.003s sys 0.005s
19:31:29.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.134785 3323 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:31:29.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.136130 3306 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
19:31:29.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.136451 3306 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/data,/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/wal
19:31:29.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82752565aafb44ceb7ba8b34162177c9"
19:31:29.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:29 on dist-test-slave-fr74"
19:31:29.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.136782 3306 fs_report.cc:383] FS layout report
19:31:29.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:31:29.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/wal
19:31:29.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/wal
19:31:29.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/data/data
19:31:29.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:31:29.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:31:29.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:31:29.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:31:29.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:31:29.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:31:29.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:31:29.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:31:29.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:31:29.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:31:29.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:31:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.190383 3306 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:31:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.196558 3306 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:31:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.196966 3306 kserver.cc:163] Server-wide thread pool size limit: 3276
19:31:29.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.290203 3306 rpc_server.cc:307] RPC server started. Bound to: 127.3.51.254:41867
19:31:29.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.290300 3374 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.51.254:41867 every 8 connection(s)
19:31:29.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.293031 3306 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/data/info.pb
19:31:29.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.298640 3375 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:31:29.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.303272 3279 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu as pid 3306
19:31:29.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.303812 3279 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/wal/instance
19:31:29.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.313755 3375 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.51.254" port: 41867 } has no permanent_uuid. Determining permanent_uuid...
19:31:29.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.3.51.253@KRBTEST.COM; defaulting to no policy
19:31:29.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:31:29.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.3.51.253@KRBTEST.COM" created.
19:31:29.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:29 dist-test-slave-fr74 krb5kdc[3292](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790623889, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.51.254@KRBTEST.COM for kudu/127.3.51.254@KRBTEST.COM
19:31:29.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:31:29.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.51.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kudu_127.3.51.253.keytab.
19:31:29.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.51.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kudu_127.3.51.253.keytab.
19:31:29.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.369220 3279 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:29.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:29.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/wal
19:31:29.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/data
19:31:29.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:31:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:31:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:31:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/logs
19:31:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/data/info.pb
19:31:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:31:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.51.253:45857
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.51.253
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kudu_127.3.51.253.keytab
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.51.253
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.51.212:37303
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.51.254:41867,127.3.51.253:45857,127.3.51.252:44579
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43921/jwks.json
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:31:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.370760 3375 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } has no permanent_uuid. Determining permanent_uuid...
19:31:29.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:29.374008 3325 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.51.253:45857: connect: Connection refused (error 111)
19:31:29.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:29.379683 3375 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.51.253:45857: Network error: Client connection negotiation failed: client connection to 127.3.51.253:45857: connect: Connection refused (error 111)
19:31:29.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.429306 3375 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } attempt: 1
19:31:29.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:29.433699 3375 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.51.253:45857: Network error: Client connection negotiation failed: client connection to 127.3.51.253:45857: connect: Connection refused (error 111)
19:31:29.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.502146 3375 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } attempt: 2
19:31:29.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:29.507041 3375 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.51.253:45857: Network error: Client connection negotiation failed: client connection to 127.3.51.253:45857: connect: Connection refused (error 111)
19:31:29.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.598553 3375 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } attempt: 3
19:31:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:29.603713 3375 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.51.253:45857: Network error: Client connection negotiation failed: client connection to 127.3.51.253:45857: connect: Connection refused (error 111)
19:31:29.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:29.709764 3381 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:31:29.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:29.710311 3381 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:31:29.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:29.710907 3381 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:31:29.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:29.745330 3381 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:31:29.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:29.745595 3381 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:31:29.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:29.745878 3381 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:31:29.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:29.746101 3381 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:31:29.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:29.746338 3381 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:31:29.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:29.746544 3381 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:31:29.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:29.746688 3381 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43921/jwks.json
19:31:29.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.747155 3375 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } attempt: 4
19:31:29.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:29.750854 3375 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.51.253:45857: Network error: Client connection negotiation failed: client connection to 127.3.51.253:45857: connect: Connection refused (error 111)
19:31:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.793815 3381 master_runner.cc:386] Master server non-default flags:
19:31:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.51.212:37303
19:31:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:31:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/data
19:31:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/wal
19:31:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:31:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:31:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.51.254:41867,127.3.51.253:45857,127.3.51.252:44579
19:31:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kudu_127.3.51.253.keytab
19:31:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.51.253
19:31:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:31:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.51.253:45857
19:31:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:31:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:31:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:31:29.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43921/jwks.json
19:31:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:31:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:31:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:31:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/data/info.pb
19:31:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.51.253
19:31:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:31:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:31:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:31:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:31:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/logs
19:31:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:31:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:31:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:31:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:31:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:31:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:31:29.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:31:29.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:31:29.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:31:29.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.795425 3381 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:31:29.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.796837 3381 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:31:29.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:29.818958 3389 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:30.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:30.050344 3375 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } attempt: 5
19:31:30.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:30.054737 3375 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.51.253:45857: Network error: Client connection negotiation failed: client connection to 127.3.51.253:45857: connect: Connection refused (error 111)
19:31:30.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:30.602540 3375 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } attempt: 6
19:31:30.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:30.610594 3375 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.51.253:45857: Network error: Client connection negotiation failed: client connection to 127.3.51.253:45857: connect: Connection refused (error 111)
19:31:31.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:29.819828 3390 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:31.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:29.822793 3381 server_base.cc:1034] running on GCE node
19:31:31.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:29.820506 3392 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.118165 3381 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:31:31.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.120823 3381 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:31:31.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.122287 3381 hybrid_clock.cc:648] HybridClock initialized: now 1790623891122256 us; error 37 us; skew 500 ppm
19:31:31.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.123309 3381 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:31:31.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:31 dist-test-slave-fr74 krb5kdc[3292](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790623891, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.51.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:31:31.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.135191 3381 init.cc:377] Logged in from keytab as kudu/127.3.51.253@KRBTEST.COM (short username kudu)
19:31:31.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.140842 3381 webserver.cc:460] Webserver started at http://127.3.51.253:38867/ using document root <none> and password file <none>
19:31:31.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.141813 3381 fs_manager.cc:359] Metadata directory not provided
19:31:31.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.142052 3381 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:31:31.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.142583 3381 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:31:31.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.147396 3381 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/data/instance:
19:31:31.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b425215a29d24263a94010d6a818b12b"
19:31:31.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:31 on dist-test-slave-fr74"
19:31:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.148826 3381 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/wal/instance:
19:31:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b425215a29d24263a94010d6a818b12b"
19:31:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:31 on dist-test-slave-fr74"
19:31:31.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.156667 3381 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
19:31:31.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.162921 3403 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:31:31.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.164058 3381 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
19:31:31.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.164462 3381 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/data,/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/wal
19:31:31.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b425215a29d24263a94010d6a818b12b"
19:31:31.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:31 on dist-test-slave-fr74"
19:31:31.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.164814 3381 fs_report.cc:383] FS layout report
19:31:31.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:31:31.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/wal
19:31:31.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/wal
19:31:31.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/data/data
19:31:31.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:31:31.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:31:31.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:31:31.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:31:31.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:31:31.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:31:31.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:31:31.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:31:31.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:31:31.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:31:31.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:31:31.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.213150 3381 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:31:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.220952 3381 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:31:31.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.221817 3381 kserver.cc:163] Server-wide thread pool size limit: 3276
19:31:31.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.315932 3381 rpc_server.cc:307] RPC server started. Bound to: 127.3.51.253:45857
19:31:31.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.316432 3455 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.51.253:45857 every 8 connection(s)
19:31:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.319008 3381 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/data/info.pb
19:31:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.322299 3279 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu as pid 3381
19:31:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.322777 3279 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/wal/instance
19:31:31.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.325538 3456 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:31:31.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.3.51.252@KRBTEST.COM; defaulting to no policy
19:31:31.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.340071 3456 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.51.254" port: 41867 } has no permanent_uuid. Determining permanent_uuid...
19:31:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:31:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.3.51.252@KRBTEST.COM" created.
19:31:31.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:31 dist-test-slave-fr74 krb5kdc[3292](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790623891, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.51.253@KRBTEST.COM for kudu/127.3.51.254@KRBTEST.COM
19:31:31.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:31:31.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.51.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kudu_127.3.51.252.keytab.
19:31:31.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.51.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kudu_127.3.51.252.keytab.
19:31:31.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.378086 3279 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:31.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:31.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/wal
19:31:31.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/data
19:31:31.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:31:31.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:31:31.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:31:31.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/logs
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/data/info.pb
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.51.252:44579
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.51.252
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kudu_127.3.51.252.keytab
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.51.252
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.51.212:37303
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:31:31.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:31:31.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.51.254:41867,127.3.51.253:45857,127.3.51.252:44579
19:31:31.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:31:31.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43921/jwks.json
19:31:31.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:31:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.383311 3456 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } has no permanent_uuid. Determining permanent_uuid...
19:31:31.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:31 dist-test-slave-fr74 krb5kdc[3292](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790623891, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.51.253@KRBTEST.COM for kudu/127.3.51.253@KRBTEST.COM
19:31:31.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.412436 3456 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } has no permanent_uuid. Determining permanent_uuid...
19:31:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:31.414827 3405 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.51.252:44579: connect: Connection refused (error 111)
19:31:31.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:31.419483 3456 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.51.252:44579: Network error: Client connection negotiation failed: client connection to 127.3.51.252:44579: connect: Connection refused (error 111)
19:31:31.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.469023 3456 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } attempt: 1
19:31:31.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:31.474424 3456 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.51.252:44579: Network error: Client connection negotiation failed: client connection to 127.3.51.252:44579: connect: Connection refused (error 111)
19:31:31.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.542899 3456 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } attempt: 2
19:31:31.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:31.547747 3456 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.51.252:44579: Network error: Client connection negotiation failed: client connection to 127.3.51.252:44579: connect: Connection refused (error 111)
19:31:31.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.639248 3456 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } attempt: 3
19:31:31.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:31.644285 3456 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.51.252:44579: Network error: Client connection negotiation failed: client connection to 127.3.51.252:44579: connect: Connection refused (error 111)
19:31:31.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.671006 3375 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } attempt: 7
19:31:31.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:31 dist-test-slave-fr74 krb5kdc[3292](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790623889, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.51.254@KRBTEST.COM for kudu/127.3.51.253@KRBTEST.COM
19:31:31.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.704288 3375 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } has no permanent_uuid. Determining permanent_uuid...
19:31:31.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:31.709952 3375 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.51.252:44579: Network error: Client connection negotiation failed: client connection to 127.3.51.252:44579: connect: Connection refused (error 111)
19:31:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:31.726049 3462 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:31:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:31.726663 3462 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:31:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:31.727255 3462 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:31:31.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:31.759562 3462 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:31:31.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:31.759843 3462 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:31:31.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:31.760228 3462 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:31:31.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:31.760553 3462 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:31:31.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:31.760861 3462 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:31:31.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:31.761175 3462 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:31:31.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:31.761406 3462 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43921/jwks.json
19:31:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.768381 3375 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } attempt: 1
19:31:31.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:31.772691 3375 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.51.252:44579: Network error: Client connection negotiation failed: client connection to 127.3.51.252:44579: connect: Connection refused (error 111)
19:31:31.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.787703 3456 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } attempt: 4
19:31:31.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:31.792160 3456 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.51.252:44579: Network error: Client connection negotiation failed: client connection to 127.3.51.252:44579: connect: Connection refused (error 111)
19:31:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.803776 3462 master_runner.cc:386] Master server non-default flags:
19:31:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.51.212:37303
19:31:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:31:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/data
19:31:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/wal
19:31:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:31:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:31:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.51.254:41867,127.3.51.253:45857,127.3.51.252:44579
19:31:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kudu_127.3.51.252.keytab
19:31:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.51.252
19:31:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:31:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.51.252:44579
19:31:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:31:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:31:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:31:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43921/jwks.json
19:31:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:31:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:31:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:31:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/data/info.pb
19:31:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.51.252
19:31:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:31:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:31:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:31:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:31:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/logs
19:31:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:31.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:31:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:31:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:31:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:31:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:31:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:31:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:31:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:31:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:31:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.805279 3462 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:31:31.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.806555 3462 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:31:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:31.827373 3473 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:31.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.854235 3375 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } attempt: 2
19:31:31.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:31.860260 3375 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.51.252:44579: Network error: Client connection negotiation failed: client connection to 127.3.51.252:44579: connect: Connection refused (error 111)
19:31:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.945721 3375 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } attempt: 3
19:31:31.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:31.950541 3375 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.51.252:44579: Network error: Client connection negotiation failed: client connection to 127.3.51.252:44579: connect: Connection refused (error 111)
19:31:32.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:32.091583 3456 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } attempt: 5
19:31:32.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:32.090965 3375 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } attempt: 4
19:31:32.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:32.096431 3456 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.51.252:44579: Network error: Client connection negotiation failed: client connection to 127.3.51.252:44579: connect: Connection refused (error 111)
19:31:32.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:32.098853 3375 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.51.252:44579: Network error: Client connection negotiation failed: client connection to 127.3.51.252:44579: connect: Connection refused (error 111)
19:31:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:32.382285 3375 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } attempt: 5
19:31:32.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:32.385804 3375 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.51.252:44579: Network error: Client connection negotiation failed: client connection to 127.3.51.252:44579: connect: Connection refused (error 111)
19:31:32.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:32.643831 3456 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } attempt: 6
19:31:32.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:32.650817 3456 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.51.252:44579: Network error: Client connection negotiation failed: client connection to 127.3.51.252:44579: connect: Connection refused (error 111)
19:31:32.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:32.938369 3375 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } attempt: 6
19:31:32.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:32.945803 3375 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.51.252:44579: Network error: Client connection negotiation failed: client connection to 127.3.51.252:44579: connect: Connection refused (error 111)
19:31:33.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:31.829980 3462 server_base.cc:1034] running on GCE node
19:31:33.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:31.827545 3472 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:33.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:31.828475 3475 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:33.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.129557 3462 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:31:33.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.132308 3462 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:31:33.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.133756 3462 hybrid_clock.cc:648] HybridClock initialized: now 1790623893133711 us; error 41 us; skew 500 ppm
19:31:33.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.134802 3462 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:31:33.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:33 dist-test-slave-fr74 krb5kdc[3292](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790623893, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.51.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:31:33.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.141237 3462 init.cc:377] Logged in from keytab as kudu/127.3.51.252@KRBTEST.COM (short username kudu)
19:31:33.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.147217 3462 webserver.cc:460] Webserver started at http://127.3.51.252:46273/ using document root <none> and password file <none>
19:31:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.148249 3462 fs_manager.cc:359] Metadata directory not provided
19:31:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.148492 3462 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:31:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.148998 3462 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:31:33.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.156247 3462 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/data/instance:
19:31:33.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ab97824899d7438e9265ba3ae41da0bc"
19:31:33.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:33 on dist-test-slave-fr74"
19:31:33.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.157585 3462 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/wal/instance:
19:31:33.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ab97824899d7438e9265ba3ae41da0bc"
19:31:33.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:33 on dist-test-slave-fr74"
19:31:33.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.165398 3462 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.000s sys 0.008s
19:31:33.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.173332 3488 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:31:33.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.174747 3462 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
19:31:33.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.175216 3462 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/data,/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/wal
19:31:33.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ab97824899d7438e9265ba3ae41da0bc"
19:31:33.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:33 on dist-test-slave-fr74"
19:31:33.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.175665 3462 fs_report.cc:383] FS layout report
19:31:33.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:31:33.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/wal
19:31:33.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/wal
19:31:33.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/data/data
19:31:33.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:31:33.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:31:33.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:31:33.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:31:33.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:31:33.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:31:33.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:31:33.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:31:33.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:31:33.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:31:33.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:31:33.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.226586 3462 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:31:33.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.233956 3462 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:31:33.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.234457 3462 kserver.cc:163] Server-wide thread pool size limit: 3276
19:31:33.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.333642 3462 rpc_server.cc:307] RPC server started. Bound to: 127.3.51.252:44579
19:31:33.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.333679 3539 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.51.252:44579 every 8 connection(s)
19:31:33.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.337555 3462 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/data/info.pb
19:31:33.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.343276 3279 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu as pid 3462
19:31:33.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.343652 3279 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/wal/instance
19:31:33.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.344333 3540 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:31:33.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.361477 3540 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.51.254" port: 41867 } has no permanent_uuid. Determining permanent_uuid...
19:31:33.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:33 dist-test-slave-fr74 krb5kdc[3292](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790623887, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.3.51.252@KRBTEST.COM
19:31:33.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:33 dist-test-slave-fr74 krb5kdc[3292](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790623893, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.51.252@KRBTEST.COM for kudu/127.3.51.254@KRBTEST.COM
19:31:33.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.418303 3540 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } has no permanent_uuid. Determining permanent_uuid...
19:31:33.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:33 dist-test-slave-fr74 krb5kdc[3292](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790623887, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.3.51.253@KRBTEST.COM
19:31:33.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:33 dist-test-slave-fr74 krb5kdc[3292](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790623893, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.51.252@KRBTEST.COM for kudu/127.3.51.253@KRBTEST.COM
19:31:33.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.446727 3540 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } has no permanent_uuid. Determining permanent_uuid...
19:31:33.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:33 dist-test-slave-fr74 krb5kdc[3292](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790623887, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.3.51.254@KRBTEST.COM
19:31:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:33 dist-test-slave-fr74 krb5kdc[3292](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790623893, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.51.252@KRBTEST.COM for kudu/127.3.51.252@KRBTEST.COM
19:31:33.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.488799 3279 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
19:31:33.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.491989 3540 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: Bootstrap starting.
19:31:33.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.500751 3540 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: Neither blocks nor log segments found. Creating new log.
19:31:33.501 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:31:33.502 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
19:31:33.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.502846 3540 log.cc:824] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: Log is configured to *not* fsync() on all Append() calls
19:31:33.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.509099 3540 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: No bootstrap required, opened a new log
19:31:33.514 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
19:31:33.515 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
19:31:33.518 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.3.51.254"
port: 41867
}
bound_http_address {
host: "127.3.51.254"
port: 42115
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.3.51.253"
port: 45857
}
bound_http_address {
host: "127.3.51.253"
port: 38867
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.3.51.252"
port: 44579
}
bound_http_address {
host: "127.3.51.252"
port: 46273
}
}
}
19:31:33.518 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
19:31:33.518 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:31:33.519 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
19:31:33.520 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
19:31:33.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.532222 3540 raft_consensus.cc:348] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82752565aafb44ceb7ba8b34162177c9" member_type: VOTER last_known_addr { host: "127.3.51.254" port: 41867 } } peers { permanent_uuid: "b425215a29d24263a94010d6a818b12b" member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } } peers { permanent_uuid: "ab97824899d7438e9265ba3ae41da0bc" member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } }
19:31:33.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.532881 3540 raft_consensus.cc:374] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:31:33.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.533175 3540 raft_consensus.cc:729] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ab97824899d7438e9265ba3ae41da0bc, State: Initialized, Role: FOLLOWER
19:31:33.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.533941 3540 consensus_queue.cc:260] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82752565aafb44ceb7ba8b34162177c9" member_type: VOTER last_known_addr { host: "127.3.51.254" port: 41867 } } peers { permanent_uuid: "b425215a29d24263a94010d6a818b12b" member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } } peers { permanent_uuid: "ab97824899d7438e9265ba3ae41da0bc" member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } }
19:31:33.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.537425 3551 sys_catalog.cc:455] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82752565aafb44ceb7ba8b34162177c9" member_type: VOTER last_known_addr { host: "127.3.51.254" port: 41867 } } peers { permanent_uuid: "b425215a29d24263a94010d6a818b12b" member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } } peers { permanent_uuid: "ab97824899d7438e9265ba3ae41da0bc" member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } } }
19:31:33.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.538434 3551 sys_catalog.cc:458] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [sys.catalog]: This master's current role is: FOLLOWER
19:31:33.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.539383 3540 sys_catalog.cc:564] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [sys.catalog]: configured and running, proceeding with master startup.
19:31:33.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:33 dist-test-slave-fr74 krb5kdc[3292](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790623893, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:31:33.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
19:31:33.554 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:31:33.558 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/krb5cc
19:31:33.566 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
19:31:33.572 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/krb5cc
19:31:33.574 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
19:31:33.579 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
19:31:33.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:33.581182 3563 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:33.582 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
19:31:33.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:33.582533 3563 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:33.582 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
19:31:33.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.567359 3462 master_runner.cc:186] Error getting master registration for 127.3.51.254:41867: OK, instance_id { permanent_uuid: "82752565aafb44ceb7ba8b34162177c9" instance_seqno: 1790623889266583 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
19:31:33.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.604379 3462 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:31:33.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:33 dist-test-slave-fr74 krb5kdc[3292](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790623893, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:31:33.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
19:31:33.622 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:31:33.622 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/krb5cc
19:31:33.624 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
19:31:33.624 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.3.51.252:44579
19:31:33.626 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
19:31:33.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.627197 3279 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu with pid 3462
19:31:33.654 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:31:33.655 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.3.51.253:45857
19:31:33.655 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
19:31:33.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.655956 3279 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu with pid 3381
19:31:33.679 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:31:33.679 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.3.51.254:41867
19:31:33.680 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
19:31:33.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.680516 3279 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu with pid 3306
19:31:33.704 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:31:33.705 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.3.51.252:44579
19:31:33.706 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
19:31:33.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:33.708164 3279 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:33.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:33.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/wal
19:31:33.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/data
19:31:33.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:31:33.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:31:33.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:31:33.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:33.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:33.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:33.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/logs
19:31:33.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/data/info.pb
19:31:33.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:33.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:31:33.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:31:33.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:31:33.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:31:33.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:33.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:33.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:33.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:31:33.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:31:33.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:31:33.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:31:33.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.51.252:44579
19:31:33.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.51.252
19:31:33.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46273
19:31:33.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kudu_127.3.51.252.keytab
19:31:33.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.51.252
19:31:33.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:31:33.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:31:33.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:31:33.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.51.212:37303
19:31:33.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:33.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:33.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:31:33.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:31:33.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.51.254:41867,127.3.51.253:45857,127.3.51.252:44579
19:31:33.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:31:33.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43921/jwks.json
19:31:33.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:31:34.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:34.043292 3567 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:31:34.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:34.043915 3567 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:31:34.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:34.044487 3567 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:31:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:34.079399 3567 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:31:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:34.079641 3567 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:31:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:34.079990 3567 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:31:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:34.080214 3567 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:31:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:34.080470 3567 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:31:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:34.080675 3567 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:31:34.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:34.080874 3567 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43921/jwks.json
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:34.122512 3567 master_runner.cc:386] Master server non-default flags:
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.51.212:37303
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/data
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/wal
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.51.254:41867,127.3.51.253:45857,127.3.51.252:44579
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kudu_127.3.51.252.keytab
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.51.252
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.51.252:44579
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43921/jwks.json
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/data/info.pb
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.51.252
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46273
19:31:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:31:34.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:34.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:31:34.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:31:34.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:34.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/logs
19:31:34.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:34.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:31:34.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:31:34.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:31:34.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:31:34.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:31:34.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:31:34.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:31:34.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:31:34.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:31:34.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:34.124040 3567 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:31:34.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:34.125015 3567 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:31:34.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:34.145228 3577 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:35.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:34.146636 3567 server_base.cc:1034] running on GCE node
19:31:35.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:34.145759 3574 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:35.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:34.145217 3575 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.434305 3567 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:31:35.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.437621 3567 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:31:35.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.439060 3567 hybrid_clock.cc:648] HybridClock initialized: now 1790623895439020 us; error 44 us; skew 500 ppm
19:31:35.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.439963 3567 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:31:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:35 dist-test-slave-fr74 krb5kdc[3292](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790623895, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.51.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:31:35.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.446211 3567 init.cc:377] Logged in from keytab as kudu/127.3.51.252@KRBTEST.COM (short username kudu)
19:31:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.451854 3567 webserver.cc:460] Webserver started at http://127.3.51.252:46273/ using document root <none> and password file <none>
19:31:35.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.452880 3567 fs_manager.cc:359] Metadata directory not provided
19:31:35.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.453158 3567 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:31:35.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.462024 3567 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
19:31:35.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.466426 3586 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:31:35.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.467576 3567 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
19:31:35.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.467949 3567 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/data,/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/wal
19:31:35.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ab97824899d7438e9265ba3ae41da0bc"
19:31:35.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:33 on dist-test-slave-fr74"
19:31:35.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.470249 3567 fs_report.cc:383] FS layout report
19:31:35.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:31:35.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/wal
19:31:35.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/wal
19:31:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/data/data
19:31:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:31:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:31:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:31:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:31:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:31:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:31:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:31:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:31:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:31:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:31:35.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:31:35.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.510329 3567 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:31:35.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.516315 3567 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:31:35.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.516860 3567 kserver.cc:163] Server-wide thread pool size limit: 3276
19:31:35.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.597749 3567 rpc_server.cc:307] RPC server started. Bound to: 127.3.51.252:44579
19:31:35.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.597868 3637 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.51.252:44579 every 8 connection(s)
19:31:35.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.600914 3567 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-2/data/info.pb
19:31:35.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.610936 3279 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu as pid 3567
19:31:35.611 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:31:35.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.612316 3638 sys_catalog.cc:263] Verifying existing consensus state
19:31:35.613 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.3.51.253:45857
19:31:35.613 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
19:31:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.614821 3279 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/wal
19:31:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/data
19:31:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:31:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:31:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:31:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/logs
19:31:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/data/info.pb
19:31:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:31:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:31:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:31:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:31:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:31:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:31:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:31:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:31:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.51.253:45857
19:31:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.51.253
19:31:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38867
19:31:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kudu_127.3.51.253.keytab
19:31:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.51.253
19:31:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:31:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:31:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:31:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.51.212:37303
19:31:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:31:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:31:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.51.254:41867,127.3.51.253:45857,127.3.51.252:44579
19:31:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:31:35.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43921/jwks.json
19:31:35.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:31:35.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.618680 3638 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: Bootstrap starting.
19:31:35.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.662521 3638 log.cc:824] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: Log is configured to *not* fsync() on all Append() calls
19:31:35.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.667413 3638 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
19:31:35.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.668233 3638 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: Bootstrap complete.
19:31:35.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.694974 3638 raft_consensus.cc:348] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82752565aafb44ceb7ba8b34162177c9" member_type: VOTER last_known_addr { host: "127.3.51.254" port: 41867 } } peers { permanent_uuid: "b425215a29d24263a94010d6a818b12b" member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } } peers { permanent_uuid: "ab97824899d7438e9265ba3ae41da0bc" member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } }
19:31:35.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.697325 3638 raft_consensus.cc:374] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:31:35.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.697525 3638 raft_consensus.cc:729] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ab97824899d7438e9265ba3ae41da0bc, State: Initialized, Role: FOLLOWER
19:31:35.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.698190 3638 consensus_queue.cc:260] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82752565aafb44ceb7ba8b34162177c9" member_type: VOTER last_known_addr { host: "127.3.51.254" port: 41867 } } peers { permanent_uuid: "b425215a29d24263a94010d6a818b12b" member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } } peers { permanent_uuid: "ab97824899d7438e9265ba3ae41da0bc" member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } }
19:31:35.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.700973 3642 sys_catalog.cc:455] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82752565aafb44ceb7ba8b34162177c9" member_type: VOTER last_known_addr { host: "127.3.51.254" port: 41867 } } peers { permanent_uuid: "b425215a29d24263a94010d6a818b12b" member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } } peers { permanent_uuid: "ab97824899d7438e9265ba3ae41da0bc" member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } } }
19:31:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.701797 3642 sys_catalog.cc:458] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [sys.catalog]: This master's current role is: FOLLOWER
19:31:35.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.702646 3638 sys_catalog.cc:564] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [sys.catalog]: configured and running, proceeding with master startup.
19:31:35.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:35.732156 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:35.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:35.732515 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.729959 3567 master_runner.cc:186] Error getting master registration for 127.3.51.254:41867: Network error: Client connection negotiation failed: client connection to 127.3.51.254:41867: connect: Connection refused (error 111),
19:31:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.756047 3567 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:31:35.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:35.772606 3592 tablet.cc:2369] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:31:35.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.805392 3642 raft_consensus.cc:482] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:31:35.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.805943 3642 raft_consensus.cc:504] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82752565aafb44ceb7ba8b34162177c9" member_type: VOTER last_known_addr { host: "127.3.51.254" port: 41867 } } peers { permanent_uuid: "b425215a29d24263a94010d6a818b12b" member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } } peers { permanent_uuid: "ab97824899d7438e9265ba3ae41da0bc" member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } }
19:31:35.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.809032 3642 leader_election.cc:288] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82752565aafb44ceb7ba8b34162177c9 (127.3.51.254:41867), b425215a29d24263a94010d6a818b12b (127.3.51.253:45857)
19:31:35.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:35.810500 3590 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.51.254:41867: connect: Connection refused (error 111)
19:31:35.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:35.816928 3590 leader_election.cc:334] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 82752565aafb44ceb7ba8b34162177c9 (127.3.51.254:41867): Network error: Client connection negotiation failed: client connection to 127.3.51.254:41867: connect: Connection refused (error 111)
19:31:35.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:35.818420 3588 leader_election.cc:334] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b425215a29d24263a94010d6a818b12b (127.3.51.253:45857): Network error: Client connection negotiation failed: client connection to 127.3.51.253:45857: connect: Connection refused (error 111)
19:31:35.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.819000 3588 leader_election.cc:302] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ab97824899d7438e9265ba3ae41da0bc; no voters: 82752565aafb44ceb7ba8b34162177c9, b425215a29d24263a94010d6a818b12b
19:31:35.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:35.819994 3642 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:31:35.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:35.958261 3640 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:31:35.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:35.958837 3640 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:31:35.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:35.960037 3640 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:31:36.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:36.005339 3640 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:31:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:36.005856 3640 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:31:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:36.006150 3640 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:31:36.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:36.006650 3640 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:31:36.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:36.006896 3640 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:31:36.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:36.007089 3640 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:31:36.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:36.007234 3640 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43921/jwks.json
19:31:36.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:36.050544 3640 master_runner.cc:386] Master server non-default flags:
19:31:36.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:36.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.51.212:37303
19:31:36.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:36.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:31:36.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/data
19:31:36.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/wal
19:31:36.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:31:36.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:31:36.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.51.254:41867,127.3.51.253:45857,127.3.51.252:44579
19:31:36.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kudu_127.3.51.253.keytab
19:31:36.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.51.253
19:31:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:31:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.51.253:45857
19:31:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:31:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:31:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:31:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43921/jwks.json
19:31:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:31:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:31:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:31:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/data/info.pb
19:31:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.51.253
19:31:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38867
19:31:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:31:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:31:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:31:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/logs
19:31:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:31:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:31:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:31:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:31:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:31:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:31:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:31:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:31:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:31:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:36.051746 3640 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:31:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:36.053061 3640 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:31:36.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:36.075001 3665 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:36.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:36.742327 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:36.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:36.742877 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:37.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:36.076882 3667 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:37.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:36.078706 3640 server_base.cc:1034] running on GCE node
19:31:37.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:36.075958 3664 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:37.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.403824 3640 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:31:37.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.407073 3640 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:31:37.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.408524 3640 hybrid_clock.cc:648] HybridClock initialized: now 1790623897408481 us; error 42 us; skew 500 ppm
19:31:37.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.409612 3640 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:31:37.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:37 dist-test-slave-fr74 krb5kdc[3292](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790623897, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.51.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:31:37.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.424777 3640 init.cc:377] Logged in from keytab as kudu/127.3.51.253@KRBTEST.COM (short username kudu)
19:31:37.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.431092 3640 webserver.cc:460] Webserver started at http://127.3.51.253:38867/ using document root <none> and password file <none>
19:31:37.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.432363 3640 fs_manager.cc:359] Metadata directory not provided
19:31:37.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.432638 3640 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:31:37.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.444125 3640 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.004s sys 0.002s
19:31:37.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.449294 3677 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:31:37.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.450618 3640 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
19:31:37.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.451031 3640 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/data,/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/wal
19:31:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b425215a29d24263a94010d6a818b12b"
19:31:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:31 on dist-test-slave-fr74"
19:31:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.454593 3640 fs_report.cc:383] FS layout report
19:31:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:31:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/wal
19:31:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/wal
19:31:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/data/data
19:31:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:31:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:31:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:31:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:31:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:31:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:31:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:31:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:31:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:31:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:31:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:31:37.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.506208 3640 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:31:37.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.514838 3640 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:31:37.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.515450 3640 kserver.cc:163] Server-wide thread pool size limit: 3276
19:31:37.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.626040 3640 rpc_server.cc:307] RPC server started. Bound to: 127.3.51.253:45857
19:31:37.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.626168 3728 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.51.253:45857 every 8 connection(s)
19:31:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.629595 3640 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/data/info.pb
19:31:37.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.639899 3279 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu as pid 3640
19:31:37.641 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:31:37.642 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.3.51.254:41867
19:31:37.642 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
19:31:37.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.645017 3279 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:37.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu
19:31:37.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/wal
19:31:37.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/data
19:31:37.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:31:37.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:31:37.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:31:37.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:37.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:37.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:37.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/logs
19:31:37.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/data/info.pb
19:31:37.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:37.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:31:37.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:31:37.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:31:37.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:31:37.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:37.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:37.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:37.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:31:37.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:31:37.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:31:37.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:31:37.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.51.254:41867
19:31:37.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.51.254
19:31:37.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42115
19:31:37.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kudu_127.3.51.254.keytab
19:31:37.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.51.254
19:31:37.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:31:37.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:31:37.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:31:37.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.51.212:37303
19:31:37.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:37.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:37.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:31:37.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:31:37.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.51.254:41867,127.3.51.253:45857,127.3.51.252:44579
19:31:37.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:31:37.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43921/jwks.json
19:31:37.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:31:37.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.650826 3729 sys_catalog.cc:263] Verifying existing consensus state
19:31:37.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260928 19:31:37.652483 3729 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:31:37.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.653789 3640 master.cc:537] Master@127.3.51.253:45857 shutting down...
19:31:37.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:37.736490 3640 master.cc:552] Master@127.3.51.253:45857 shutdown complete.
19:31:37.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:37.746047 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:37.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:37.746342 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:37.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:31:38.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:38.034287 3731 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:31:38.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:38.035050 3731 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:31:38.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:38.035801 3731 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:31:38.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:38.047513 3736 raft_consensus.cc:482] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:31:38.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:38.047843 3736 raft_consensus.cc:504] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82752565aafb44ceb7ba8b34162177c9" member_type: VOTER last_known_addr { host: "127.3.51.254" port: 41867 } } peers { permanent_uuid: "b425215a29d24263a94010d6a818b12b" member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } } peers { permanent_uuid: "ab97824899d7438e9265ba3ae41da0bc" member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } }
19:31:38.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:38.050083 3736 leader_election.cc:288] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82752565aafb44ceb7ba8b34162177c9 (127.3.51.254:41867), b425215a29d24263a94010d6a818b12b (127.3.51.253:45857)
19:31:38.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:38.055460 3590 leader_election.cc:334] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 82752565aafb44ceb7ba8b34162177c9 (127.3.51.254:41867): Network error: Client connection negotiation failed: client connection to 127.3.51.254:41867: connect: Connection refused (error 111)
19:31:38.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:38.056485 3588 leader_election.cc:334] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b425215a29d24263a94010d6a818b12b (127.3.51.253:45857): Network error: Client connection negotiation failed: client connection to 127.3.51.253:45857: connect: Connection refused (error 111)
19:31:38.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:38.056808 3588 leader_election.cc:302] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ab97824899d7438e9265ba3ae41da0bc; no voters: 82752565aafb44ceb7ba8b34162177c9, b425215a29d24263a94010d6a818b12b
19:31:38.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:38.057451 3736 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:31:38.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:38.075794 3731 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:31:38.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:38.076083 3731 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:31:38.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:38.076540 3731 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:31:38.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:38.076884 3731 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:31:38.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:38.077206 3731 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:31:38.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:38.077567 3731 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:31:38.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:38.077827 3731 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43921/jwks.json
19:31:38.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:38.118034 3731 master_runner.cc:386] Master server non-default flags:
19:31:38.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:31:38.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.51.212:37303
19:31:38.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:31:38.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:31:38.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/data
19:31:38.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/wal
19:31:38.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:31:38.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:31:38.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.51.254:41867,127.3.51.253:45857,127.3.51.252:44579
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/krb5kdc/kudu_127.3.51.254.keytab
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.51.254
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.51.254:41867
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43921/jwks.json
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/data/info.pb
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.51.254
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42115
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:31:38.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/logs
19:31:38.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:31:38.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:31:38.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:31:38.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:31:38.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:31:38.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:31:38.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:31:38.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:31:38.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:31:38.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:31:38.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:38.119544 3731 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:31:38.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:38.120759 3731 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:31:38.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:38.139797 3742 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:38.751866 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:38.752353 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:39.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:38.139981 3741 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:39.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:38.141362 3744 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:31:39.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:39.474728 3743 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1332 milliseconds
19:31:39.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.474854 3731 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:31:39.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.476279 3731 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:31:39.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.479137 3731 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:31:39.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.480645 3731 hybrid_clock.cc:648] HybridClock initialized: now 1790623899480595 us; error 53 us; skew 500 ppm
19:31:39.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.482015 3731 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:31:39.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:39 dist-test-slave-fr74 krb5kdc[3292](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790623899, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.51.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:31:39.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.489640 3731 init.cc:377] Logged in from keytab as kudu/127.3.51.254@KRBTEST.COM (short username kudu)
19:31:39.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.498571 3731 webserver.cc:460] Webserver started at http://127.3.51.254:42115/ using document root <none> and password file <none>
19:31:39.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.499797 3731 fs_manager.cc:359] Metadata directory not provided
19:31:39.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.500061 3731 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:31:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.510918 3731 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.006s sys 0.000s
19:31:39.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.516332 3752 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:31:39.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.517488 3731 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
19:31:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.517771 3731 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/data,/tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/wal
19:31:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82752565aafb44ceb7ba8b34162177c9"
19:31:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:31:29 on dist-test-slave-fr74"
19:31:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.520045 3731 fs_report.cc:383] FS layout report
19:31:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:31:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/wal
19:31:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/wal
19:31:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/data/data
19:31:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:31:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:31:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:31:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:31:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:31:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:31:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:31:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:31:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:31:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:31:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:31:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.572009 3731 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:31:39.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.579559 3731 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:31:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.580168 3731 kserver.cc:163] Server-wide thread pool size limit: 3276
19:31:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.667923 3731 rpc_server.cc:307] RPC server started. Bound to: 127.3.51.254:41867
19:31:39.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.668007 3804 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.51.254:41867 every 8 connection(s)
19:31:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.671613 3731 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/data/info.pb
19:31:39.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.681157 3279 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu as pid 3731
19:31:39.681 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:31:39.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.684578 3805 sys_catalog.cc:263] Verifying existing consensus state
19:31:39.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260928 19:31:39.685322 3805 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:31:39.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.685782 3731 master.cc:537] Master@127.3.51.254:41867 shutting down...
19:31:39.745 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.51.252:44579(127.3.51.252:44579)] connecting to peer
19:31:39.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:39.756595 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:39.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:39.756946 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:39.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:39.766659 3731 master.cc:552] Master@127.3.51.254:41867 shutdown complete.
19:31:39.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskDtkeQs/test-tmp/mini-kudu-cluster7728581177159865662/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:31:39.818 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:31:39.819 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:31:39.878 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.3.51.252:44579(127.3.51.252:44579)] Successfully connected to peer
19:31:39.880 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:39.880 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:39.882 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:39.940 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:39.941 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:39.942 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:39.959 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
19:31:39.969 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.3.51.252
19:31:40.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:40 dist-test-slave-fr74 krb5kdc[3292](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790623893, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.3.51.252@KRBTEST.COM
19:31:40.246 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xd4cbef58, L:/127.0.0.1:46536 - R:/127.3.51.252:44579] using SASL/GSSAPI
19:31:40.262 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:31:40.269 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:31:40.272 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.273 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.273 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:31:40.274 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.278 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.278 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.279 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.279 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:31:40.287 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:31:40.288 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:31:40.291 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.291 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.293 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.294 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.294 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.296 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.296 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:31:40.307 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:31:40.308 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:31:40.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.310 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.311 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.314 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.314 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.315 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.315 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:31:40.327 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:31:40.329 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:31:40.330 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.330 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.331 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.333 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.333 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.334 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.334 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:31:40.367 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:31:40.368 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:31:40.373 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.373 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.373 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.374 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.374 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.375 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.376 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:31:40.448 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:31:40.448 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:31:40.451 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.451 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.455 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.455 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.457 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.462 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:31:40.527 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:31:40.528 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:31:40.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.530 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.530 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.532 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.533 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:31:40.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:40.630995 3812 raft_consensus.cc:482] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:31:40.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:40.631832 3812 raft_consensus.cc:504] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82752565aafb44ceb7ba8b34162177c9" member_type: VOTER last_known_addr { host: "127.3.51.254" port: 41867 } } peers { permanent_uuid: "b425215a29d24263a94010d6a818b12b" member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } } peers { permanent_uuid: "ab97824899d7438e9265ba3ae41da0bc" member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } }
19:31:40.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:40.634043 3812 leader_election.cc:288] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82752565aafb44ceb7ba8b34162177c9 (127.3.51.254:41867), b425215a29d24263a94010d6a818b12b (127.3.51.253:45857)
19:31:40.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:40.656502 3590 leader_election.cc:334] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 82752565aafb44ceb7ba8b34162177c9 (127.3.51.254:41867): Network error: Client connection negotiation failed: client connection to 127.3.51.254:41867: connect: Connection refused (error 111)
19:31:40.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:40.658984 3588 leader_election.cc:334] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b425215a29d24263a94010d6a818b12b (127.3.51.253:45857): Network error: Client connection negotiation failed: client connection to 127.3.51.253:45857: connect: Connection refused (error 111)
19:31:40.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:40.659637 3588 leader_election.cc:302] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ab97824899d7438e9265ba3ae41da0bc; no voters: 82752565aafb44ceb7ba8b34162177c9, b425215a29d24263a94010d6a818b12b
19:31:40.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:40.660410 3812 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:31:40.687 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:31:40.688 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:31:40.690 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.690 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.690 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.690 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.691 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:40.692 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:40.695 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:31:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:40.758338 3567 master_runner.cc:186] Error getting master registration for 127.3.51.254:41867: Network error: Client connection negotiation failed: client connection to 127.3.51.254:41867: connect: Connection refused (error 111),
19:31:40.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:40.758810 3567 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:31:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:40.760124 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:40.760344 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:41.047 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:31:41.050 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:31:41.050 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:41.050 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:41.051 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:41.053 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:41.053 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:41.053 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:41.054 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:31:41.466 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:31:41.468 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:31:41.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:41.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:41.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:41.470 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:41.471 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:41.471 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:41.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:31:41.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:41.763096 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:41.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:41.763391 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:41.987 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:31:41.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:41.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:41.990 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:41.990 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:31:41.993 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:41.993 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:41.994 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:41.994 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:31:42.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:42.766585 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:42.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:42.766824 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:43.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:43.769853 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:43.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:43.770138 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:44.147 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:31:44.149 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:31:44.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:44.150 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:44.151 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:44.151 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:44.151 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:44.151 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:44.152 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:31:44.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:44.773861 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:44.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:44.774207 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:45.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:45.168036 3821 raft_consensus.cc:482] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:31:45.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:45.168506 3821 raft_consensus.cc:504] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82752565aafb44ceb7ba8b34162177c9" member_type: VOTER last_known_addr { host: "127.3.51.254" port: 41867 } } peers { permanent_uuid: "b425215a29d24263a94010d6a818b12b" member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } } peers { permanent_uuid: "ab97824899d7438e9265ba3ae41da0bc" member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } }
19:31:45.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:45.170135 3821 leader_election.cc:288] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82752565aafb44ceb7ba8b34162177c9 (127.3.51.254:41867), b425215a29d24263a94010d6a818b12b (127.3.51.253:45857)
19:31:45.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:45.173149 3588 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.51.253:45857: connect: Connection refused (error 111) [suppressed 5 similar messages]
19:31:45.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:45.176661 3590 leader_election.cc:334] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 82752565aafb44ceb7ba8b34162177c9 (127.3.51.254:41867): Network error: Client connection negotiation failed: client connection to 127.3.51.254:41867: connect: Connection refused (error 111)
19:31:45.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:45.177253 3588 leader_election.cc:334] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b425215a29d24263a94010d6a818b12b (127.3.51.253:45857): Network error: Client connection negotiation failed: client connection to 127.3.51.253:45857: connect: Connection refused (error 111)
19:31:45.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:45.177696 3588 leader_election.cc:302] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ab97824899d7438e9265ba3ae41da0bc; no voters: 82752565aafb44ceb7ba8b34162177c9, b425215a29d24263a94010d6a818b12b
19:31:45.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:45.178464 3821 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:31:45.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:45 dist-test-slave-fr74 krb5kdc[3292](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790623905, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.51.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:31:45.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:45.450207 3582 init.cc:295] Successfully reacquired a new kerberos TGT
19:31:45.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:45.761747 3567 master_runner.cc:186] Error getting master registration for 127.3.51.254:41867: Network error: Client connection negotiation failed: client connection to 127.3.51.254:41867: connect: Connection refused (error 111),
19:31:45.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:45.762153 3567 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:31:45.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:45.777709 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:45.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:45.778039 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:46.467 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:31:46.467 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:31:46.470 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:46.470 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:46.471 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:46.471 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:46.471 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:46.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:46.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:31:46.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:46.781023 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:46.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:46.781312 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:46.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:46.796489 3827 raft_consensus.cc:482] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:31:46.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:46.796963 3827 raft_consensus.cc:504] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82752565aafb44ceb7ba8b34162177c9" member_type: VOTER last_known_addr { host: "127.3.51.254" port: 41867 } } peers { permanent_uuid: "b425215a29d24263a94010d6a818b12b" member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } } peers { permanent_uuid: "ab97824899d7438e9265ba3ae41da0bc" member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } }
19:31:46.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:46.798559 3827 leader_election.cc:288] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82752565aafb44ceb7ba8b34162177c9 (127.3.51.254:41867), b425215a29d24263a94010d6a818b12b (127.3.51.253:45857)
19:31:46.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:46.804118 3590 leader_election.cc:334] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 82752565aafb44ceb7ba8b34162177c9 (127.3.51.254:41867): Network error: Client connection negotiation failed: client connection to 127.3.51.254:41867: connect: Connection refused (error 111)
19:31:46.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:46.804823 3588 leader_election.cc:334] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b425215a29d24263a94010d6a818b12b (127.3.51.253:45857): Network error: Client connection negotiation failed: client connection to 127.3.51.253:45857: connect: Connection refused (error 111)
19:31:46.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:46.805119 3588 leader_election.cc:302] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ab97824899d7438e9265ba3ae41da0bc; no voters: 82752565aafb44ceb7ba8b34162177c9, b425215a29d24263a94010d6a818b12b
19:31:46.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:46.805598 3827 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:31:47.347 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:31:47.350 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:31:47.350 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:47.350 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:47.351 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:47.353 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:47.353 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:47.353 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:47.353 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:31:47.427 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:31:47.429 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:31:47.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:47.430 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:47.431 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:47.434 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:47.434 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:47.434 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:47.434 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:31:47.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:47.784701 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:47.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:47.785012 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:47.967 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:31:47.968 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:31:47.971 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:47.971 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:47.971 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:47.971 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:47.972 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:47.972 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:47.973 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:31:48.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:48.715531 3832 raft_consensus.cc:482] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:31:48.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:48.715886 3832 raft_consensus.cc:504] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82752565aafb44ceb7ba8b34162177c9" member_type: VOTER last_known_addr { host: "127.3.51.254" port: 41867 } } peers { permanent_uuid: "b425215a29d24263a94010d6a818b12b" member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } } peers { permanent_uuid: "ab97824899d7438e9265ba3ae41da0bc" member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } }
19:31:48.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:48.717258 3832 leader_election.cc:288] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82752565aafb44ceb7ba8b34162177c9 (127.3.51.254:41867), b425215a29d24263a94010d6a818b12b (127.3.51.253:45857)
19:31:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:48.723107 3588 leader_election.cc:334] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b425215a29d24263a94010d6a818b12b (127.3.51.253:45857): Network error: Client connection negotiation failed: client connection to 127.3.51.253:45857: connect: Connection refused (error 111)
19:31:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:48.723610 3590 leader_election.cc:334] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 82752565aafb44ceb7ba8b34162177c9 (127.3.51.254:41867): Network error: Client connection negotiation failed: client connection to 127.3.51.254:41867: connect: Connection refused (error 111)
19:31:48.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:48.723874 3590 leader_election.cc:302] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ab97824899d7438e9265ba3ae41da0bc; no voters: 82752565aafb44ceb7ba8b34162177c9, b425215a29d24263a94010d6a818b12b
19:31:48.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:48.724380 3832 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:31:48.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:48.787935 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:48.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:48.788225 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:49.791615 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:49.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:49.791884 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:50.227 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:31:50.228 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:31:50.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:50.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:50.230 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:50.232 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:50.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:50.232 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:50.232 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:31:50.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:50.764812 3567 master_runner.cc:186] Error getting master registration for 127.3.51.254:41867: Network error: Client connection negotiation failed: client connection to 127.3.51.254:41867: connect: Connection refused (error 111),
19:31:50.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:50.765179 3567 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:31:50.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:50.795789 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:50.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:50.796113 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:51.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:51.799501 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:51.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:51.799809 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:52.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:52.803308 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:52.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:52.804112 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:52.927 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:31:52.929 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:31:52.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:52.930 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:52.930 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:52.930 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:52.931 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:52.931 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:52.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:31:53.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:53.807427 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:53.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:53.807685 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:54.547 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:31:54.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:54.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:54.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:54.549 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:31:54.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:54.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:54.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:54.552 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:31:54.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:54.811080 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:54.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:54.811425 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:55.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:31:55 dist-test-slave-fr74 krb5kdc[3292](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790623915, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.51.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:31:55.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:55.452867 3582 init.cc:295] Successfully reacquired a new kerberos TGT
19:31:55.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:55.768272 3567 master_runner.cc:186] Error getting master registration for 127.3.51.254:41867: Network error: Client connection negotiation failed: client connection to 127.3.51.254:41867: connect: Connection refused (error 111),
19:31:55.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:55.768668 3567 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:31:55.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:55.814265 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:55.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:55.814530 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:56.666 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:31:56.667 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:31:56.667 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:56.668 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:56.668 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:56.668 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:56.668 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:56.668 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:56.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:31:56.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:56.817349 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:56.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:56.817595 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:57.647 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:31:57.650 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:57.651 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:57.651 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:31:57.651 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:31:57.653 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:57.653 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:57.653 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:31:57.653 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:31:57.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:57.820609 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:57.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:57.820883 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:58.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:58.138919 3847 raft_consensus.cc:482] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:31:58.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:58.139484 3847 raft_consensus.cc:504] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82752565aafb44ceb7ba8b34162177c9" member_type: VOTER last_known_addr { host: "127.3.51.254" port: 41867 } } peers { permanent_uuid: "b425215a29d24263a94010d6a818b12b" member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } } peers { permanent_uuid: "ab97824899d7438e9265ba3ae41da0bc" member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } }
19:31:58.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:58.141258 3847 leader_election.cc:288] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82752565aafb44ceb7ba8b34162177c9 (127.3.51.254:41867), b425215a29d24263a94010d6a818b12b (127.3.51.253:45857)
19:31:58.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:58.144068 3590 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.51.254:41867: connect: Connection refused (error 111) [suppressed 5 similar messages]
19:31:58.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:58.146692 3588 leader_election.cc:334] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b425215a29d24263a94010d6a818b12b (127.3.51.253:45857): Network error: Client connection negotiation failed: client connection to 127.3.51.253:45857: connect: Connection refused (error 111)
19:31:58.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:58.147239 3590 leader_election.cc:334] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 82752565aafb44ceb7ba8b34162177c9 (127.3.51.254:41867): Network error: Client connection negotiation failed: client connection to 127.3.51.254:41867: connect: Connection refused (error 111)
19:31:58.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:58.147883 3590 leader_election.cc:302] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ab97824899d7438e9265ba3ae41da0bc; no voters: 82752565aafb44ceb7ba8b34162177c9, b425215a29d24263a94010d6a818b12b
19:31:58.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:31:58.148607 3847 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:31:58.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:58.823817 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:58.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:58.824095 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:31:59.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:59.827272 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:31:59.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:31:59.827616 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:32:00.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:32:00.771423 3567 master_runner.cc:186] Error getting master registration for 127.3.51.254:41867: Network error: Client connection negotiation failed: client connection to 127.3.51.254:41867: connect: Connection refused (error 111),
19:32:00.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:32:00.771989 3567 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:32:00.788 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:32:00.788 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:32:00.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:32:00.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:32:00.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:32:00.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:32:00.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:32:00.790 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:32:00.791 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:32:00.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:32:00.830500 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:32:00.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:32:00.830818 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:32:01.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:32:01.834849 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:32:01.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:32:01.835177 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:32:02.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:32:02.601255 3858 raft_consensus.cc:482] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:32:02.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:32:02.601693 3858 raft_consensus.cc:504] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "82752565aafb44ceb7ba8b34162177c9" member_type: VOTER last_known_addr { host: "127.3.51.254" port: 41867 } } peers { permanent_uuid: "b425215a29d24263a94010d6a818b12b" member_type: VOTER last_known_addr { host: "127.3.51.253" port: 45857 } } peers { permanent_uuid: "ab97824899d7438e9265ba3ae41da0bc" member_type: VOTER last_known_addr { host: "127.3.51.252" port: 44579 } }
19:32:02.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:32:02.603878 3858 leader_election.cc:288] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 82752565aafb44ceb7ba8b34162177c9 (127.3.51.254:41867), b425215a29d24263a94010d6a818b12b (127.3.51.253:45857)
19:32:02.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:32:02.610827 3588 leader_election.cc:334] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b425215a29d24263a94010d6a818b12b (127.3.51.253:45857): Network error: Client connection negotiation failed: client connection to 127.3.51.253:45857: connect: Connection refused (error 111)
19:32:02.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:32:02.611255 3590 leader_election.cc:334] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 82752565aafb44ceb7ba8b34162177c9 (127.3.51.254:41867): Network error: Client connection negotiation failed: client connection to 127.3.51.254:41867: connect: Connection refused (error 111)
19:32:02.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:32:02.611624 3588 leader_election.cc:302] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ab97824899d7438e9265ba3ae41da0bc; no voters: 82752565aafb44ceb7ba8b34162177c9, b425215a29d24263a94010d6a818b12b
19:32:02.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:32:02.612339 3858 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:32:02.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:32:02.838758 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:32:02.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:32:02.839035 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:32:02.867 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:32:02.868 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:32:02.868 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:32:02.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:32:02.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:32:02.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:32:02.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:32:02.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:32:02.871 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:32:03.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:32:03.841857 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:32:03.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:32:03.842166 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:32:04.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:32:04.845844 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:32:04.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:32:04.846226 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:32:04.967 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:32:04.968 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:32:04.968 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:32:04.968 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:32:04.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:32:04.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:32:04.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:32:04.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:32:04.972 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:32:05.247 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:32:05.247 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:32:05.248 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:32:05.248 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:32:05.248 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:32:05.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:32:05.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:32:05.249 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:32:05.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:32:05.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:32:05 dist-test-slave-fr74 krb5kdc[3292](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790623925, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.51.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:32:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:32:05.455272 3582 init.cc:295] Successfully reacquired a new kerberos TGT
19:32:05.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:32:05.775298 3567 master_runner.cc:186] Error getting master registration for 127.3.51.254:41867: Network error: Client connection negotiation failed: client connection to 127.3.51.254:41867: connect: Connection refused (error 111),
19:32:05.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:32:05.775771 3567 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:32:05.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:32:05.849396 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:32:05.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:32:05.849648 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:32:06.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:32:06.852049 3653 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ab97824899d7438e9265ba3ae41da0bc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:32:06.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:32:06.852295 3653 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:32:07.667 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.253:45857(127.3.51.253:45857)] connecting to peer
19:32:07.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:32:07.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.51.253:45857(127.3.51.253:45857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:32:07.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.253:45857: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857
19:32:07.669 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.51.254:41867(127.3.51.254:41867)] connecting to peer
19:32:07.670 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:32:07.670 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.254:41867(127.3.51.254:41867)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:32:07.670 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.51.254:41867: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
19:32:07.671 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867; will retry
19:32:07.677 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=27987), Traces: [0ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [127ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [190ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [250ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [567ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [572ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [582ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [586ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [587ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [589ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [600ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [602ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [603ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [606ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [619ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [622ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [625ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [635ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [639ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [641ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [642ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [643ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [675ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [682ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [683ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [684ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [685ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [760ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [763ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [764ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [771ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [835ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [839ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [840ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [841ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [842ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [995ms] refreshing cache from master, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [999ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [1003ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [1004ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1355ms] refreshing cache from master, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [1359ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [1360ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [1362ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1363ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1775ms] refreshing cache from master, [1775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [1775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [1777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [1778ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [1780ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1781ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [1781ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [2295ms] refreshing cache from master, [2296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [2296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [2299ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [2299ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [2302ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [2303ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [2303ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [4455ms] refreshing cache from master, [4456ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [4456ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [4457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [4459ms] too many traces: truncated at 100 traces, deferred=Deferred@895107142(state=PENDING, result=null, callback=(continuation of Deferred@378007409 after retry RPC after error@1878257999) -> (continuation of Deferred@1204730739 after retry RPC after error@544316348) -> (continuation of Deferred@161748132 after retry RPC after error@2041443670) -> (continuation of Deferred@1267391539 after retry RPC after error@1728713349) -> (continuation of Deferred@1319255028 after retry RPC after error@1479766067) -> (continuation of Deferred@1170555970 after retry RPC after error@1928128581) -> (continuation of Deferred@768479351 after retry RPC after error@1849646869) -> (continuation of Deferred@28832196 after retry RPC after error@344756087) -> (continuation of Deferred@1226598457 after retry RPC after error@1573863928) -> (continuation of Deferred@1674097236 after retry RPC after error@323972717) -> (continuation of Deferred@960004509 after retry RPC after error@509597635) -> (continuation of Deferred@950246478 after retry RPC after error@1350021425) -> (continuation of Deferred@759097035 after retry RPC after error@1296026976) -> (continuation of Deferred@1775925605 after retry RPC after error@80327510) -> (continuation of Deferred@1578833338 after retry RPC after error@1836877780) -> (continuation of Deferred@1473759823 after retry RPC after error@508395744) -> (continuation of Deferred@410906710 after retry RPC after error@297174698) -> (continuation of Deferred@2019210081 after retry RPC after error@174224906) -> (continuation of Deferred@2112538704 after retry RPC after error@1489801873) -> (continuation of Deferred@1973482322 after retry RPC after error@1818954186) -> (continuation of Deferred@1206445151 after retry RPC after error@2064428585) -> (continuation of Deferred@155510905 after retry RPC after error@1029014978) -> (continuation of Deferred@1033847851 after retry RPC after error@595380311) -> (continuation of Deferred@232123233 after retry RPC after error@482284684) -> (continuation of Deferred@1531356460 after retry RPC after error@515764323) -> (continuation of Deferred@1007787574 after retry RPC after error@1483667020), errback=(continuation of Deferred@378007409 after retry RPC after error@1878257999) -> (continuation of Deferred@1204730739 after retry RPC after error@544316348) -> (continuation of Deferred@161748132 after retry RPC after error@2041443670) -> (continuation of Deferred@1267391539 after retry RPC after error@1728713349) -> (continuation of Deferred@1319255028 after retry RPC after error@1479766067) -> (continuation of Deferred@1170555970 after retry RPC after error@1928128581) -> (continuation of Deferred@768479351 after retry RPC after error@1849646869) -> (continuation of Deferred@28832196 after retry RPC after error@344756087) -> (continuation of Deferred@1226598457 after retry RPC after error@1573863928) -> (continuation of Deferred@1674097236 after retry RPC after error@323972717) -> (continuation of Deferred@960004509 after retry RPC after error@509597635) -> (continuation of Deferred@950246478 after retry RPC after error@1350021425) -> (continuation of Deferred@759097035 after retry RPC after error@1296026976) -> (continuation of Deferred@1775925605 after retry RPC after error@80327510) -> (continuation of Deferred@1578833338 after retry RPC after error@1836877780) -> (continuation of Deferred@1473759823 after retry RPC after error@508395744) -> (continuation of Deferred@410906710 after retry RPC after error@297174698) -> (continuation of Deferred@2019210081 after retry RPC after error@174224906) -> (continuation of Deferred@2112538704 after retry RPC after error@1489801873) -> (continuation of Deferred@1973482322 after retry RPC after error@1818954186) -> (continuation of Deferred@1206445151 after retry RPC after error@2064428585) -> (continuation of Deferred@155510905 after retry RPC after error@1029014978) -> (continuation of Deferred@1033847851 after retry RPC after error@595380311) -> (continuation of Deferred@232123233 after retry RPC after error@482284684) -> (continuation of Deferred@1531356460 after retry RPC after error@515764323) -> (continuation of Deferred@1007787574 after retry RPC after error@1483667020)))
19:32:07.683 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.3.51.254:41867(127.3.51.254:41867)] disconnecting while in state TERMINATED
19:32:07.684 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.3.51.252:44579(127.3.51.252:44579)] disconnecting while in state READY
19:32:07.685 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.3.51.253:45857(127.3.51.253:45857)] disconnecting while in state TERMINATED
19:32:07.690 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
19:32:07.691 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
19:32:07.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:32:07.694741 3279 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu with pid 3567
19:32:07.696 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.3.51.252:44579(127.3.51.252:44579)] handling channelInactive
19:32:07.696 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.51.252:44579(127.3.51.252:44579)] cleaning up while in state READY due to: connection closed
19:32:08.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T19:32:08Z chronyd exiting
19:32:09.945 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
19:32:09.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:32:09.946700 2552 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu with pid 2834
19:32:09.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:32:09.973136 2552 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu with pid 2991
19:32:10.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:32:10.003299 2552 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu with pid 3137
19:32:10.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:32:10.034438 2552 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu with pid 2564
19:32:10.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:32:10.064553 2552 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu with pid 2634
19:32:10.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:32:10.094184 2552 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDtkeQs/build/tsan/bin/kudu with pid 2709
19:32:11.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T19:32:11Z chronyd exiting
19:32:12.309 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=27987), Traces: [0ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [127ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [190ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [250ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [567ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [572ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [582ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [586ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [587ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [589ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [600ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [602ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [603ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [606ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [619ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [622ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [625ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [635ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [639ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [641ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [642ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [643ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [675ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [682ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [683ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [684ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [685ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [760ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [763ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [764ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [771ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [835ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [839ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [840ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [841ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [842ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [995ms] refreshing cache from master, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [999ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [1003ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [1004ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1355ms] refreshing cache from master, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [1359ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [1360ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [1362ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1363ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1775ms] refreshing cache from master, [1775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [1775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [1777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [1778ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [1780ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1781ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [1781ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [2295ms] refreshing cache from master, [2296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [2296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [2299ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [2299ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [2302ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [2303ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [2303ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [4455ms] refreshing cache from master, [4456ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [4456ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [4457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [4459ms] too many traces: truncated at 100 traces, deferred=Deferred@895107142(state=PENDING, result=null, callback=(continuation of Deferred@378007409 after retry RPC after error@1878257999) -> (continuation of Deferred@1204730739 after retry RPC after error@544316348) -> (continuation of Deferred@161748132 after retry RPC after error@2041443670) -> (continuation of Deferred@1267391539 after retry RPC after error@1728713349) -> (continuation of Deferred@1319255028 after retry RPC after error@1479766067) -> (continuation of Deferred@1170555970 after retry RPC after error@1928128581) -> (continuation of Deferred@768479351 after retry RPC after error@1849646869) -> (continuation of Deferred@28832196 after retry RPC after error@344756087) -> (continuation of Deferred@1226598457 after retry RPC after error@1573863928) -> (continuation of Deferred@1674097236 after retry RPC after error@323972717) -> (continuation of Deferred@960004509 after retry RPC after error@509597635) -> (continuation of Deferred@950246478 after retry RPC after error@1350021425) -> (continuation of Deferred@759097035 after retry RPC after error@1296026976) -> (continuation of Deferred@1775925605 after retry RPC after error@80327510) -> (continuation of Deferred@1578833338 after retry RPC after error@1836877780) -> (continuation of Deferred@1473759823 after retry RPC after error@508395744) -> (continuation of Deferred@410906710 after retry RPC after error@297174698) -> (continuation of Deferred@2019210081 after retry RPC after error@174224906) -> (continuation of Deferred@2112538704 after retry RPC after error@1489801873) -> (continuation of Deferred@1973482322 after retry RPC after error@1818954186) -> (continuation of Deferred@1206445151 after retry RPC after error@2064428585) -> (continuation of Deferred@155510905 after retry RPC after error@1029014978) -> (continuation of Deferred@1033847851 after retry RPC after error@595380311) -> (continuation of Deferred@232123233 after retry RPC after error@482284684) -> (continuation of Deferred@1531356460 after retry RPC after error@515764323) -> (continuation of Deferred@1007787574 after retry RPC after error@1483667020), errback=(continuation of Deferred@378007409 after retry RPC after error@1878257999) -> (continuation of Deferred@1204730739 after retry RPC after error@544316348) -> (continuation of Deferred@161748132 after retry RPC after error@2041443670) -> (continuation of Deferred@1267391539 after retry RPC after error@1728713349) -> (continuation of Deferred@1319255028 after retry RPC after error@1479766067) -> (continuation of Deferred@1170555970 after retry RPC after error@1928128581) -> (continuation of Deferred@768479351 after retry RPC after error@1849646869) -> (continuation of Deferred@28832196 after retry RPC after error@344756087) -> (continuation of Deferred@1226598457 after retry RPC after error@1573863928) -> (continuation of Deferred@1674097236 after retry RPC after error@323972717) -> (continuation of Deferred@960004509 after retry RPC after error@509597635) -> (continuation of Deferred@950246478 after retry RPC after error@1350021425) -> (continuation of Deferred@759097035 after retry RPC after error@1296026976) -> (continuation of Deferred@1775925605 after retry RPC after error@80327510) -> (continuation of Deferred@1578833338 after retry RPC after error@1836877780) -> (continuation of Deferred@1473759823 after retry RPC after error@508395744) -> (continuation of Deferred@410906710 after retry RPC after error@297174698) -> (continuation of Deferred@2019210081 after retry RPC after error@174224906) -> (continuation of Deferred@2112538704 after retry RPC after error@1489801873) -> (continuation of Deferred@1973482322 after retry RPC after error@1818954186) -> (continuation of Deferred@1206445151 after retry RPC after error@2064428585) -> (continuation of Deferred@155510905 after retry RPC after error@1029014978) -> (continuation of Deferred@1033847851 after retry RPC after error@595380311) -> (continuation of Deferred@232123233 after retry RPC after error@482284684) -> (continuation of Deferred@1531356460 after retry RPC after error@515764323) -> (continuation of Deferred@1007787574 after retry RPC after error@1483667020)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.51.254:41867
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=27987), Traces: [0ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [127ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [190ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [250ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [567ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [572ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [582ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [586ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [587ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [589ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [600ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [602ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [603ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [606ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [619ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [622ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [625ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [635ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [639ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [641ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [642ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [643ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [675ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [682ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [683ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [684ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [685ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [760ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [763ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [764ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [771ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [835ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [839ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [840ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [841ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [842ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [995ms] refreshing cache from master, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [999ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [1003ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [1004ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1355ms] refreshing cache from master, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [1359ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [1360ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [1362ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1363ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1775ms] refreshing cache from master, [1775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [1775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [1777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [1778ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [1780ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1781ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [1781ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [2295ms] refreshing cache from master, [2296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [2296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [2299ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [2299ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [2302ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [2303ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [2303ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [4455ms] refreshing cache from master, [4456ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [4456ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [4457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [4459ms] too many traces: truncated at 100 traces, deferred=Deferred@895107142(state=PENDING, result=null, callback=(continuation of Deferred@378007409 after retry RPC after error@1878257999) -> (continuation of Deferred@1204730739 after retry RPC after error@544316348) -> (continuation of Deferred@161748132 after retry RPC after error@2041443670) -> (continuation of Deferred@1267391539 after retry RPC after error@1728713349) -> (continuation of Deferred@1319255028 after retry RPC after error@1479766067) -> (continuation of Deferred@1170555970 after retry RPC after error@1928128581) -> (continuation of Deferred@768479351 after retry RPC after error@1849646869) -> (continuation of Deferred@28832196 after retry RPC after error@344756087) -> (continuation of Deferred@1226598457 after retry RPC after error@1573863928) -> (continuation of Deferred@1674097236 after retry RPC after error@323972717) -> (continuation of Deferred@960004509 after retry RPC after error@509597635) -> (continuation of Deferred@950246478 after retry RPC after error@1350021425) -> (continuation of Deferred@759097035 after retry RPC after error@1296026976) -> (continuation of Deferred@1775925605 after retry RPC after error@80327510) -> (continuation of Deferred@1578833338 after retry RPC after error@1836877780) -> (continuation of Deferred@1473759823 after retry RPC after error@508395744) -> (continuation of Deferred@410906710 after retry RPC after error@297174698) -> (continuation of Deferred@2019210081 after retry RPC after error@174224906) -> (continuation of Deferred@2112538704 after retry RPC after error@1489801873) -> (continuation of Deferred@1973482322 after retry RPC after error@1818954186) -> (continuation of Deferred@1206445151 after retry RPC after error@2064428585) -> (continuation of Deferred@155510905 after retry RPC after error@1029014978) -> (continuation of Deferred@1033847851 after retry RPC after error@595380311) -> (continuation of Deferred@232123233 after retry RPC after error@482284684) -> (continuation of Deferred@1531356460 after retry RPC after error@515764323) -> (continuation of Deferred@1007787574 after retry RPC after error@1483667020), errback=(continuation of Deferred@378007409 after retry RPC after error@1878257999) -> (continuation of Deferred@1204730739 after retry RPC after error@544316348) -> (continuation of Deferred@161748132 after retry RPC after error@2041443670) -> (continuation of Deferred@1267391539 after retry RPC after error@1728713349) -> (continuation of Deferred@1319255028 after retry RPC after error@1479766067) -> (continuation of Deferred@1170555970 after retry RPC after error@1928128581) -> (continuation of Deferred@768479351 after retry RPC after error@1849646869) -> (continuation of Deferred@28832196 after retry RPC after error@344756087) -> (continuation of Deferred@1226598457 after retry RPC after error@1573863928) -> (continuation of Deferred@1674097236 after retry RPC after error@323972717) -> (continuation of Deferred@960004509 after retry RPC after error@509597635) -> (continuation of Deferred@950246478 after retry RPC after error@1350021425) -> (continuation of Deferred@759097035 after retry RPC after error@1296026976) -> (continuation of Deferred@1775925605 after retry RPC after error@80327510) -> (continuation of Deferred@1578833338 after retry RPC after error@1836877780) -> (continuation of Deferred@1473759823 after retry RPC after error@508395744) -> (continuation of Deferred@410906710 after retry RPC after error@297174698) -> (continuation of Deferred@2019210081 after retry RPC after error@174224906) -> (continuation of Deferred@2112538704 after retry RPC after error@1489801873) -> (continuation of Deferred@1973482322 after retry RPC after error@1818954186) -> (continuation of Deferred@1206445151 after retry RPC after error@2064428585) -> (continuation of Deferred@155510905 after retry RPC after error@1029014978) -> (continuation of Deferred@1033847851 after retry RPC after error@595380311) -> (continuation of Deferred@232123233 after retry RPC after error@482284684) -> (continuation of Deferred@1531356460 after retry RPC after error@515764323) -> (continuation of Deferred@1007787574 after retry RPC after error@1483667020)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.51.254:41867
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
19:32:13.093 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
19:32:13.094 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=27987), Traces: [0ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [127ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [190ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [250ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [567ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [572ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [582ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [586ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [587ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [589ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [600ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [602ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [603ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [606ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [619ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [622ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [625ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [635ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [639ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [641ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [642ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [643ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [675ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [682ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [683ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [684ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [685ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [760ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [763ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [764ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [771ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [835ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [839ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [840ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [841ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [842ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [995ms] refreshing cache from master, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [999ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [1003ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [1004ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1355ms] refreshing cache from master, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [1359ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [1360ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [1362ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1363ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1775ms] refreshing cache from master, [1775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [1775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [1777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [1778ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [1780ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1781ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [1781ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [2295ms] refreshing cache from master, [2296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [2296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [2299ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [2299ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [2302ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [2303ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [2303ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [4455ms] refreshing cache from master, [4456ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [4456ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [4457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [4459ms] too many traces: truncated at 100 traces, deferred=Deferred@895107142(state=PENDING, result=null, callback=(continuation of Deferred@378007409 after retry RPC after error@1878257999) -> (continuation of Deferred@1204730739 after retry RPC after error@544316348) -> (continuation of Deferred@161748132 after retry RPC after error@2041443670) -> (continuation of Deferred@1267391539 after retry RPC after error@1728713349) -> (continuation of Deferred@1319255028 after retry RPC after error@1479766067) -> (continuation of Deferred@1170555970 after retry RPC after error@1928128581) -> (continuation of Deferred@768479351 after retry RPC after error@1849646869) -> (continuation of Deferred@28832196 after retry RPC after error@344756087) -> (continuation of Deferred@1226598457 after retry RPC after error@1573863928) -> (continuation of Deferred@1674097236 after retry RPC after error@323972717) -> (continuation of Deferred@960004509 after retry RPC after error@509597635) -> (continuation of Deferred@950246478 after retry RPC after error@1350021425) -> (continuation of Deferred@759097035 after retry RPC after error@1296026976) -> (continuation of Deferred@1775925605 after retry RPC after error@80327510) -> (continuation of Deferred@1578833338 after retry RPC after error@1836877780) -> (continuation of Deferred@1473759823 after retry RPC after error@508395744) -> (continuation of Deferred@410906710 after retry RPC after error@297174698) -> (continuation of Deferred@2019210081 after retry RPC after error@174224906) -> (continuation of Deferred@2112538704 after retry RPC after error@1489801873) -> (continuation of Deferred@1973482322 after retry RPC after error@1818954186) -> (continuation of Deferred@1206445151 after retry RPC after error@2064428585) -> (continuation of Deferred@155510905 after retry RPC after error@1029014978) -> (continuation of Deferred@1033847851 after retry RPC after error@595380311) -> (continuation of Deferred@232123233 after retry RPC after error@482284684) -> (continuation of Deferred@1531356460 after retry RPC after error@515764323) -> (continuation of Deferred@1007787574 after retry RPC after error@1483667020), errback=(continuation of Deferred@378007409 after retry RPC after error@1878257999) -> (continuation of Deferred@1204730739 after retry RPC after error@544316348) -> (continuation of Deferred@161748132 after retry RPC after error@2041443670) -> (continuation of Deferred@1267391539 after retry RPC after error@1728713349) -> (continuation of Deferred@1319255028 after retry RPC after error@1479766067) -> (continuation of Deferred@1170555970 after retry RPC after error@1928128581) -> (continuation of Deferred@768479351 after retry RPC after error@1849646869) -> (continuation of Deferred@28832196 after retry RPC after error@344756087) -> (continuation of Deferred@1226598457 after retry RPC after error@1573863928) -> (continuation of Deferred@1674097236 after retry RPC after error@323972717) -> (continuation of Deferred@960004509 after retry RPC after error@509597635) -> (continuation of Deferred@950246478 after retry RPC after error@1350021425) -> (continuation of Deferred@759097035 after retry RPC after error@1296026976) -> (continuation of Deferred@1775925605 after retry RPC after error@80327510) -> (continuation of Deferred@1578833338 after retry RPC after error@1836877780) -> (continuation of Deferred@1473759823 after retry RPC after error@508395744) -> (continuation of Deferred@410906710 after retry RPC after error@297174698) -> (continuation of Deferred@2019210081 after retry RPC after error@174224906) -> (continuation of Deferred@2112538704 after retry RPC after error@1489801873) -> (continuation of Deferred@1973482322 after retry RPC after error@1818954186) -> (continuation of Deferred@1206445151 after retry RPC after error@2064428585) -> (continuation of Deferred@155510905 after retry RPC after error@1029014978) -> (continuation of Deferred@1033847851 after retry RPC after error@595380311) -> (continuation of Deferred@232123233 after retry RPC after error@482284684) -> (continuation of Deferred@1531356460 after retry RPC after error@515764323) -> (continuation of Deferred@1007787574 after retry RPC after error@1483667020)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.51.254:41867
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=27987), Traces: [0ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [127ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [190ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [250ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [567ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [572ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [582ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [586ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [587ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [589ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [600ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [602ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [603ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [606ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [619ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [622ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [625ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [635ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [639ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [641ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [642ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [643ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [675ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [682ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [683ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [684ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [685ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [760ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [763ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [764ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [771ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [835ms] refreshing cache from master, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [835ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [839ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [840ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [841ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [842ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [995ms] refreshing cache from master, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [996ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [999ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [1003ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [1004ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1355ms] refreshing cache from master, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [1359ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [1360ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [1362ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1363ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1775ms] refreshing cache from master, [1775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [1775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [1777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [1778ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [1780ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [1781ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [1781ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [2295ms] refreshing cache from master, [2296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [2296ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [2299ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.253:45857: Network error: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857, [2299ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [2302ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.252:44579: OK, [2303ms] Sub RPC ConnectToMaster: received response from server master-127.3.51.254:41867: Network error: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [2303ms] delaying RPC due to: Service unavailable: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867, [4455ms] refreshing cache from master, [4456ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.252:44579, [4456ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.253:45857, [4457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.51.254:41867, [4459ms] too many traces: truncated at 100 traces, deferred=Deferred@895107142(state=PENDING, result=null, callback=(continuation of Deferred@378007409 after retry RPC after error@1878257999) -> (continuation of Deferred@1204730739 after retry RPC after error@544316348) -> (continuation of Deferred@161748132 after retry RPC after error@2041443670) -> (continuation of Deferred@1267391539 after retry RPC after error@1728713349) -> (continuation of Deferred@1319255028 after retry RPC after error@1479766067) -> (continuation of Deferred@1170555970 after retry RPC after error@1928128581) -> (continuation of Deferred@768479351 after retry RPC after error@1849646869) -> (continuation of Deferred@28832196 after retry RPC after error@344756087) -> (continuation of Deferred@1226598457 after retry RPC after error@1573863928) -> (continuation of Deferred@1674097236 after retry RPC after error@323972717) -> (continuation of Deferred@960004509 after retry RPC after error@509597635) -> (continuation of Deferred@950246478 after retry RPC after error@1350021425) -> (continuation of Deferred@759097035 after retry RPC after error@1296026976) -> (continuation of Deferred@1775925605 after retry RPC after error@80327510) -> (continuation of Deferred@1578833338 after retry RPC after error@1836877780) -> (continuation of Deferred@1473759823 after retry RPC after error@508395744) -> (continuation of Deferred@410906710 after retry RPC after error@297174698) -> (continuation of Deferred@2019210081 after retry RPC after error@174224906) -> (continuation of Deferred@2112538704 after retry RPC after error@1489801873) -> (continuation of Deferred@1973482322 after retry RPC after error@1818954186) -> (continuation of Deferred@1206445151 after retry RPC after error@2064428585) -> (continuation of Deferred@155510905 after retry RPC after error@1029014978) -> (continuation of Deferred@1033847851 after retry RPC after error@595380311) -> (continuation of Deferred@232123233 after retry RPC after error@482284684) -> (continuation of Deferred@1531356460 after retry RPC after error@515764323) -> (continuation of Deferred@1007787574 after retry RPC after error@1483667020), errback=(continuation of Deferred@378007409 after retry RPC after error@1878257999) -> (continuation of Deferred@1204730739 after retry RPC after error@544316348) -> (continuation of Deferred@161748132 after retry RPC after error@2041443670) -> (continuation of Deferred@1267391539 after retry RPC after error@1728713349) -> (continuation of Deferred@1319255028 after retry RPC after error@1479766067) -> (continuation of Deferred@1170555970 after retry RPC after error@1928128581) -> (continuation of Deferred@768479351 after retry RPC after error@1849646869) -> (continuation of Deferred@28832196 after retry RPC after error@344756087) -> (continuation of Deferred@1226598457 after retry RPC after error@1573863928) -> (continuation of Deferred@1674097236 after retry RPC after error@323972717) -> (continuation of Deferred@960004509 after retry RPC after error@509597635) -> (continuation of Deferred@950246478 after retry RPC after error@1350021425) -> (continuation of Deferred@759097035 after retry RPC after error@1296026976) -> (continuation of Deferred@1775925605 after retry RPC after error@80327510) -> (continuation of Deferred@1578833338 after retry RPC after error@1836877780) -> (continuation of Deferred@1473759823 after retry RPC after error@508395744) -> (continuation of Deferred@410906710 after retry RPC after error@297174698) -> (continuation of Deferred@2019210081 after retry RPC after error@174224906) -> (continuation of Deferred@2112538704 after retry RPC after error@1489801873) -> (continuation of Deferred@1973482322 after retry RPC after error@1818954186) -> (continuation of Deferred@1206445151 after retry RPC after error@2064428585) -> (continuation of Deferred@155510905 after retry RPC after error@1029014978) -> (continuation of Deferred@1033847851 after retry RPC after error@595380311) -> (continuation of Deferred@232123233 after retry RPC after error@482284684) -> (continuation of Deferred@1531356460 after retry RPC after error@515764323) -> (continuation of Deferred@1007787574 after retry RPC after error@1483667020)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.3.51.252:44579,127.3.51.253:45857,127.3.51.254:41867) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.253:45857(127.3.51.253:45857): Connection refused: /127.3.51.253:45857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.51.254:41867(127.3.51.254:41867): Connection refused: /127.3.51.254:41867
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.51.254:41867
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)