20:00:48.707 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
20:00:48.719 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasktTLa0n/test-tmp
20:00:48.722 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:00:48.725 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasktTLa0n/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:00:48.936 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
20:00:49.124 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:00:49.125 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:00:49.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:49.128371 17984 mini_oidc.cc:152] Starting JWKS server
20:00:49.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:49.132470 17984 webserver.cc:460] Webserver started at http://127.0.0.1:35059/ using document root <none> and password file <none>
20:00:49.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:49.132993 17984 mini_oidc.cc:189] Starting OIDC Discovery server
20:00:49.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:49.134748 17984 webserver.cc:460] Webserver started at http://127.0.0.1:36029/ using document root <none> and password file <none>
20:00:49.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T20:00:49Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
20:00:49.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T20:00:49Z Disabled control of system clock
20:00:49.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:49.185397 17984 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:00:49.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:00:49.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-0/wal
20:00:49.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-0/data
20:00:49.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:00:49.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:00:49.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:00:49.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:49.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:49.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:49.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-0/logs
20:00:49.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-0/data/info.pb
20:00:49.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:00:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:00:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:00:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:00:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:00:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:00:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.144.62:42077
20:00:49.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.144.62
20:00:49.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:49.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.144.20:44579
20:00:49.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:49.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:49.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:49.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:49.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.144.62:42077,127.17.144.61:45409,127.17.144.60:42331
20:00:49.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:49.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35059/jwks.json with env {}
20:00:49.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:49.489344 17996 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:00:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:49.489895 17996 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:00:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:49.490545 17996 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:00:49.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:49.521533 17996 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:00:49.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:49.521847 17996 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:00:49.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:49.522127 17996 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:00:49.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:49.522318 17996 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:00:49.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:49.522621 17996 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:00:49.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:49.522858 17996 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35059/jwks.json
20:00:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:49.557847 17996 master_runner.cc:386] Master server non-default flags:
20:00:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.144.20:44579
20:00:49.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:49.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:49.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-0/data
20:00:49.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-0/wal
20:00:49.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:49.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.144.62:42077,127.17.144.61:45409,127.17.144.60:42331
20:00:49.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:49.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:49.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:49.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.144.62:42077
20:00:49.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:49.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:00:49.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:49.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35059/jwks.json
20:00:49.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:49.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:49.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-0/data/info.pb
20:00:49.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.144.62
20:00:49.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:49.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:00:49.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:49.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:00:49.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:00:49.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:49.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-0/logs
20:00:49.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:49.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:00:49.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:00:49.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:00:49.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:00:49.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:00:49.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:00:49.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 18:51:07 UTC on ccycloud.quasar-lbjuyh.root.comops.site
20:00:49.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456544
20:00:49.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:00:49.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:49.559194 17996 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:00:49.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:49.561333 17996 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:00:49.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:49.581133 18002 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:50.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:49.581377 18003 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:50.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:49.582971 17996 server_base.cc:1034] running on GCE node
20:00:50.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:49.582614 18005 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:50.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.779331 17996 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:00:50.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.782666 17996 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:00:50.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.784060 17996 hybrid_clock.cc:648] HybridClock initialized: now 1781899250784027 us; error 43 us; skew 500 ppm
20:00:50.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.785044 17996 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:00:50.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.797061 17996 webserver.cc:460] Webserver started at http://127.17.144.62:33523/ using document root <none> and password file <none>
20:00:50.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.798122 17996 fs_manager.cc:359] Metadata directory not provided
20:00:50.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.798369 17996 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:00:50.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.798907 17996 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:00:50.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.803773 17996 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-0/data/instance:
20:00:50.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bef6096de95f42adb4c52fe18c43f55a"
20:00:50.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:00:50 on dist-test-slave-pdss"
20:00:50.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.805171 17996 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-0/wal/instance:
20:00:50.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bef6096de95f42adb4c52fe18c43f55a"
20:00:50.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:00:50 on dist-test-slave-pdss"
20:00:50.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.812474 17996 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
20:00:50.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.818193 18012 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:00:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.819319 17996 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
20:00:50.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.819696 17996 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-0/data,/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-0/wal
20:00:50.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bef6096de95f42adb4c52fe18c43f55a"
20:00:50.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:00:50 on dist-test-slave-pdss"
20:00:50.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.820134 17996 fs_report.cc:383] FS layout report
20:00:50.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:00:50.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-0/wal
20:00:50.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-0/wal
20:00:50.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-0/data/data
20:00:50.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:00:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:00:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:00:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:00:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:00:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:00:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:00:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:00:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:00:50.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:00:50.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:00:50.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.855346 17996 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:00:50.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.857059 17996 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:00:50.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.857599 17996 kserver.cc:163] Server-wide thread pool size limit: 3276
20:00:50.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.924889 17996 rpc_server.cc:307] RPC server started. Bound to: 127.17.144.62:42077
20:00:50.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.925060 18063 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.144.62:42077 every 8 connection(s)
20:00:50.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.927726 17996 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-0/data/info.pb
20:00:50.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.931828 17984 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu as pid 17996
20:00:50.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.932359 17984 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-0/wal/instance
20:00:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.933794 18064 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:00:50.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.937232 17984 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:00:50.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:00:50.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-1/wal
20:00:50.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-1/data
20:00:50.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:00:50.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:00:50.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:00:50.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:50.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:50.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:50.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-1/logs
20:00:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-1/data/info.pb
20:00:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:00:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:00:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:00:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:00:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:50.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:00:50.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:00:50.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:50.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:50.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.144.61:45409
20:00:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.144.61
20:00:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.144.20:44579
20:00:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:50.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:50.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.144.62:42077,127.17.144.61:45409,127.17.144.60:42331
20:00:50.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:50.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35059/jwks.json with env {}
20:00:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.948482 18064 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.144.62" port: 42077 } has no permanent_uuid. Determining permanent_uuid...
20:00:50.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:50.978222 18064 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } has no permanent_uuid. Determining permanent_uuid...
20:00:50.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:50.981051 18015 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.144.61:45409: connect: Connection refused (error 111)
20:00:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:50.985178 18064 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.144.61:45409: Network error: Client connection negotiation failed: client connection to 127.17.144.61:45409: connect: Connection refused (error 111)
20:00:51.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:51.034754 18064 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } attempt: 1
20:00:51.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:51.038606 18064 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.144.61:45409: Network error: Client connection negotiation failed: client connection to 127.17.144.61:45409: connect: Connection refused (error 111)
20:00:51.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:51.107093 18064 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } attempt: 2
20:00:51.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:51.111043 18064 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.144.61:45409: Network error: Client connection negotiation failed: client connection to 127.17.144.61:45409: connect: Connection refused (error 111)
20:00:51.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:51.202528 18064 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } attempt: 3
20:00:51.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:51.206606 18064 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.144.61:45409: Network error: Client connection negotiation failed: client connection to 127.17.144.61:45409: connect: Connection refused (error 111)
20:00:51.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:51.261621 18066 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:00:51.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:51.262588 18066 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:00:51.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:51.263096 18066 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:00:51.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:51.295302 18066 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:00:51.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:51.295634 18066 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:00:51.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:51.295826 18066 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:00:51.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:51.296466 18066 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:00:51.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:51.296648 18066 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:00:51.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:51.296917 18066 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35059/jwks.json
20:00:51.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:51.333966 18066 master_runner.cc:386] Master server non-default flags:
20:00:51.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:51.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.144.20:44579
20:00:51.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:51.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:51.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-1/data
20:00:51.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-1/wal
20:00:51.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:51.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.144.62:42077,127.17.144.61:45409,127.17.144.60:42331
20:00:51.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:51.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:51.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:51.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.144.61:45409
20:00:51.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:51.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:00:51.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:51.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35059/jwks.json
20:00:51.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:51.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:51.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-1/data/info.pb
20:00:51.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.144.61
20:00:51.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:00:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:00:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:00:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-1/logs
20:00:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:00:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:00:51.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:00:51.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:00:51.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:00:51.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:00:51.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 18:51:07 UTC on ccycloud.quasar-lbjuyh.root.comops.site
20:00:51.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456544
20:00:51.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:00:51.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:51.335244 18066 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:00:51.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:51.337388 18066 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:00:51.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:51.350015 18064 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } attempt: 4
20:00:51.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:51.354786 18064 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.144.61:45409: Network error: Client connection negotiation failed: client connection to 127.17.144.61:45409: connect: Connection refused (error 111)
20:00:51.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:51.358322 18078 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:51.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:51.654239 18064 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } attempt: 5
20:00:51.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:51.658344 18064 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.144.61:45409: Network error: Client connection negotiation failed: client connection to 127.17.144.61:45409: connect: Connection refused (error 111)
20:00:52.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.206074 18064 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } attempt: 6
20:00:52.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:52.217231 18064 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.144.61:45409: Network error: Client connection negotiation failed: client connection to 127.17.144.61:45409: connect: Connection refused (error 111)
20:00:52.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:51.360613 18066 server_base.cc:1034] running on GCE node
20:00:52.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:51.358333 18075 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:52.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:51.359287 18076 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:52.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.606887 18066 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:00:52.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.609777 18066 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:00:52.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.611204 18066 hybrid_clock.cc:648] HybridClock initialized: now 1781899252611155 us; error 57 us; skew 500 ppm
20:00:52.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.612321 18066 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:00:52.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.619485 18066 webserver.cc:460] Webserver started at http://127.17.144.61:38693/ using document root <none> and password file <none>
20:00:52.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.620666 18066 fs_manager.cc:359] Metadata directory not provided
20:00:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.621651 18066 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:00:52.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.622244 18066 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:00:52.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.627274 18066 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-1/data/instance:
20:00:52.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "716ad9b58f7f4f899cc8e91c400f715c"
20:00:52.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:00:52 on dist-test-slave-pdss"
20:00:52.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.628611 18066 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-1/wal/instance:
20:00:52.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "716ad9b58f7f4f899cc8e91c400f715c"
20:00:52.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:00:52 on dist-test-slave-pdss"
20:00:52.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.636349 18066 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.005s
20:00:52.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.642577 18087 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:00:52.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.643996 18066 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
20:00:52.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.644513 18066 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-1/data,/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-1/wal
20:00:52.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "716ad9b58f7f4f899cc8e91c400f715c"
20:00:52.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:00:52 on dist-test-slave-pdss"
20:00:52.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.644992 18066 fs_report.cc:383] FS layout report
20:00:52.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:00:52.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-1/wal
20:00:52.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-1/wal
20:00:52.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-1/data/data
20:00:52.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:00:52.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:00:52.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:00:52.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:00:52.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:00:52.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:00:52.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:00:52.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:00:52.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:00:52.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:00:52.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:00:52.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.680480 18066 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:00:52.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.682271 18066 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:00:52.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.682852 18066 kserver.cc:163] Server-wide thread pool size limit: 3276
20:00:52.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.761063 18066 rpc_server.cc:307] RPC server started. Bound to: 127.17.144.61:45409
20:00:52.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.761184 18138 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.144.61:45409 every 8 connection(s)
20:00:52.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.763998 18066 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-1/data/info.pb
20:00:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.767181 17984 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu as pid 18066
20:00:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.767659 17984 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-1/wal/instance
20:00:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.770710 17984 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:00:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:00:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-2/wal
20:00:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-2/data
20:00:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:00:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:00:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:00:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-2/logs
20:00:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-2/data/info.pb
20:00:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:00:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:00:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:00:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:00:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:00:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:00:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.144.60:42331
20:00:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.144.60
20:00:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.144.20:44579
20:00:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.144.62:42077,127.17.144.61:45409,127.17.144.60:42331
20:00:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35059/jwks.json with env {}
20:00:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.770944 18139 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:00:52.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.784191 18139 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.144.62" port: 42077 } has no permanent_uuid. Determining permanent_uuid...
20:00:52.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.812726 18139 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } has no permanent_uuid. Determining permanent_uuid...
20:00:52.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.830035 18139 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } has no permanent_uuid. Determining permanent_uuid...
20:00:52.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:52.832070 18089 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.144.60:42331: connect: Connection refused (error 111)
20:00:52.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:52.836510 18139 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.144.60:42331: Network error: Client connection negotiation failed: client connection to 127.17.144.60:42331: connect: Connection refused (error 111)
20:00:52.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.886080 18139 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } attempt: 1
20:00:52.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:52.889485 18139 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.144.60:42331: Network error: Client connection negotiation failed: client connection to 127.17.144.60:42331: connect: Connection refused (error 111)
20:00:52.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:52.957800 18139 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } attempt: 2
20:00:52.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:52.961447 18139 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.144.60:42331: Network error: Client connection negotiation failed: client connection to 127.17.144.60:42331: connect: Connection refused (error 111)
20:00:53.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:53.052906 18139 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } attempt: 3
20:00:53.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:53.056609 18139 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.144.60:42331: Network error: Client connection negotiation failed: client connection to 127.17.144.60:42331: connect: Connection refused (error 111)
20:00:53.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:53.165661 18141 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:00:53.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:53.166487 18141 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:00:53.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:53.167394 18141 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:00:53.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:53.200040 18139 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } attempt: 4
20:00:53.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:53.201120 18141 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:00:53.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:53.201481 18141 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:00:53.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:53.201838 18141 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:00:53.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:53.202178 18141 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:00:53.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:53.202477 18141 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:00:53.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:53.202963 18141 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35059/jwks.json
20:00:53.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:53.204123 18139 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.144.60:42331: Network error: Client connection negotiation failed: client connection to 127.17.144.60:42331: connect: Connection refused (error 111)
20:00:53.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:53.240031 18141 master_runner.cc:386] Master server non-default flags:
20:00:53.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.144.20:44579
20:00:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-2/data
20:00:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-2/wal
20:00:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:00:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.144.62:42077,127.17.144.61:45409,127.17.144.60:42331
20:00:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:00:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.144.60:42331
20:00:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:00:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:00:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:00:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35059/jwks.json
20:00:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-2/data/info.pb
20:00:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.144.60
20:00:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:00:53.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:00:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:00:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-2/logs
20:00:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:00:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:00:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:00:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:00:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:00:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:00:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 18:51:07 UTC on ccycloud.quasar-lbjuyh.root.comops.site
20:00:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456544
20:00:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:00:53.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:53.241626 18141 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:00:53.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:53.243824 18141 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:00:53.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:53.264427 18154 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:53.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:53.278311 18064 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } attempt: 7
20:00:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:53.294144 18064 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } has no permanent_uuid. Determining permanent_uuid...
20:00:53.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:53.301229 18064 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.144.60:42331: Network error: Client connection negotiation failed: client connection to 127.17.144.60:42331: connect: Connection refused (error 111)
20:00:53.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:53.359828 18064 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } attempt: 1
20:00:53.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:53.365070 18064 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.144.60:42331: Network error: Client connection negotiation failed: client connection to 127.17.144.60:42331: connect: Connection refused (error 111)
20:00:53.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:53.446521 18064 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } attempt: 2
20:00:53.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:53.451025 18064 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.144.60:42331: Network error: Client connection negotiation failed: client connection to 127.17.144.60:42331: connect: Connection refused (error 111)
20:00:53.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:53.503600 18139 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } attempt: 5
20:00:53.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:53.508492 18139 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.144.60:42331: Network error: Client connection negotiation failed: client connection to 127.17.144.60:42331: connect: Connection refused (error 111)
20:00:53.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:53.536557 18064 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } attempt: 3
20:00:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:53.542418 18064 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.144.60:42331: Network error: Client connection negotiation failed: client connection to 127.17.144.60:42331: connect: Connection refused (error 111)
20:00:53.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:53.682874 18064 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } attempt: 4
20:00:53.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:53.687561 18064 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.144.60:42331: Network error: Client connection negotiation failed: client connection to 127.17.144.60:42331: connect: Connection refused (error 111)
20:00:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:53.970975 18064 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } attempt: 5
20:00:53.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:53.975592 18064 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.144.60:42331: Network error: Client connection negotiation failed: client connection to 127.17.144.60:42331: connect: Connection refused (error 111)
20:00:54.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.056195 18139 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } attempt: 6
20:00:54.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:54.064522 18139 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.144.60:42331: Network error: Client connection negotiation failed: client connection to 127.17.144.60:42331: connect: Connection refused (error 111)
20:00:54.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:53.264477 18152 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:54.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:53.266209 18141 server_base.cc:1034] running on GCE node
20:00:54.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:53.265962 18151 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:54.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.482795 18141 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:00:54.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.486163 18141 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:00:54.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.487663 18141 hybrid_clock.cc:648] HybridClock initialized: now 1781899254487621 us; error 69 us; skew 500 ppm
20:00:54.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.488585 18141 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:00:54.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.496403 18141 webserver.cc:460] Webserver started at http://127.17.144.60:44723/ using document root <none> and password file <none>
20:00:54.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.497413 18141 fs_manager.cc:359] Metadata directory not provided
20:00:54.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.497617 18141 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:00:54.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.498020 18141 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:00:54.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.505184 18141 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-2/data/instance:
20:00:54.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "abb75db3400c4a1784d3cb03cfc744a0"
20:00:54.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:00:54 on dist-test-slave-pdss"
20:00:54.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.506753 18141 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-2/wal/instance:
20:00:54.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "abb75db3400c4a1784d3cb03cfc744a0"
20:00:54.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:00:54 on dist-test-slave-pdss"
20:00:54.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.516352 18141 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.010s sys 0.000s
20:00:54.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.522421 18165 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:00:54.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.524329 18141 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
20:00:54.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.524696 18141 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-2/data,/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-2/wal
20:00:54.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "abb75db3400c4a1784d3cb03cfc744a0"
20:00:54.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:00:54 on dist-test-slave-pdss"
20:00:54.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.525110 18141 fs_report.cc:383] FS layout report
20:00:54.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:00:54.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-2/wal
20:00:54.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-2/wal
20:00:54.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-2/data/data
20:00:54.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:00:54.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:00:54.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:00:54.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:00:54.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:00:54.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:00:54.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:00:54.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:00:54.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:00:54.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:00:54.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:00:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.528045 18064 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } attempt: 6
20:00:54.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:54.534693 18064 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.144.60:42331: Network error: Client connection negotiation failed: client connection to 127.17.144.60:42331: connect: Connection refused (error 111)
20:00:54.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.575115 18141 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:00:54.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.577100 18141 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:00:54.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.577636 18141 kserver.cc:163] Server-wide thread pool size limit: 3276
20:00:54.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.647996 18218 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.144.60:42331 every 8 connection(s)
20:00:54.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.647867 18141 rpc_server.cc:307] RPC server started. Bound to: 127.17.144.60:42331
20:00:54.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.651279 18141 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-2/data/info.pb
20:00:54.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.657615 18219 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:00:54.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.659358 17984 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu as pid 18141
20:00:54.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.659684 17984 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/master-2/wal/instance
20:00:54.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.673285 18219 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.144.62" port: 42077 } has no permanent_uuid. Determining permanent_uuid...
20:00:54.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.705286 18219 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } has no permanent_uuid. Determining permanent_uuid...
20:00:54.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.718380 18219 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } has no permanent_uuid. Determining permanent_uuid...
20:00:54.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.736071 18219 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0: Bootstrap starting.
20:00:54.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.742699 18219 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0: Neither blocks nor log segments found. Creating new log.
20:00:54.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.744436 18219 log.cc:824] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0: Log is configured to *not* fsync() on all Append() calls
20:00:54.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.750102 18219 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0: No bootstrap required, opened a new log
20:00:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.773348 18219 raft_consensus.cc:348] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bef6096de95f42adb4c52fe18c43f55a" member_type: VOTER last_known_addr { host: "127.17.144.62" port: 42077 } } peers { permanent_uuid: "716ad9b58f7f4f899cc8e91c400f715c" member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } } peers { permanent_uuid: "abb75db3400c4a1784d3cb03cfc744a0" member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } }
20:00:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.773954 18219 raft_consensus.cc:374] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:00:54.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.774135 18219 raft_consensus.cc:729] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: abb75db3400c4a1784d3cb03cfc744a0, State: Initialized, Role: FOLLOWER
20:00:54.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.774873 18219 consensus_queue.cc:260] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bef6096de95f42adb4c52fe18c43f55a" member_type: VOTER last_known_addr { host: "127.17.144.62" port: 42077 } } peers { permanent_uuid: "716ad9b58f7f4f899cc8e91c400f715c" member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } } peers { permanent_uuid: "abb75db3400c4a1784d3cb03cfc744a0" member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } }
20:00:54.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.777542 18228 sys_catalog.cc:455] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bef6096de95f42adb4c52fe18c43f55a" member_type: VOTER last_known_addr { host: "127.17.144.62" port: 42077 } } peers { permanent_uuid: "716ad9b58f7f4f899cc8e91c400f715c" member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } } peers { permanent_uuid: "abb75db3400c4a1784d3cb03cfc744a0" member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } } }
20:00:54.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.778405 18228 sys_catalog.cc:458] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [sys.catalog]: This master's current role is: FOLLOWER
20:00:54.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.778925 18219 sys_catalog.cc:564] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [sys.catalog]: configured and running, proceeding with master startup.
20:00:54.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:54.804044 18239 catalog_manager.cc:1558] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:00:54.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:54.804345 18239 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:00:54.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:54.833050 18173 tablet.cc:2369] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:00:54.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.800503 18141 master_runner.cc:186] Error getting master registration for 127.17.144.62:42077: OK, instance_id { permanent_uuid: "bef6096de95f42adb4c52fe18c43f55a" instance_seqno: 1781899250906606 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:00:54.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:54.837093 18141 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:00:55.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.002343 18228 raft_consensus.cc:482] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:00:55.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.002924 18228 raft_consensus.cc:504] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bef6096de95f42adb4c52fe18c43f55a" member_type: VOTER last_known_addr { host: "127.17.144.62" port: 42077 } } peers { permanent_uuid: "716ad9b58f7f4f899cc8e91c400f715c" member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } } peers { permanent_uuid: "abb75db3400c4a1784d3cb03cfc744a0" member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } }
20:00:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.005138 18228 leader_election.cc:288] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bef6096de95f42adb4c52fe18c43f55a (127.17.144.62:42077), 716ad9b58f7f4f899cc8e91c400f715c (127.17.144.61:45409)
20:00:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.125125 18139 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } attempt: 7
20:00:55.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.148795 18139 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c: Bootstrap starting.
20:00:55.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.154263 18139 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c: Neither blocks nor log segments found. Creating new log.
20:00:55.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.156091 18139 log.cc:824] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c: Log is configured to *not* fsync() on all Append() calls
20:00:55.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.160984 18139 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c: No bootstrap required, opened a new log
20:00:55.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.193991 18139 raft_consensus.cc:348] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bef6096de95f42adb4c52fe18c43f55a" member_type: VOTER last_known_addr { host: "127.17.144.62" port: 42077 } } peers { permanent_uuid: "716ad9b58f7f4f899cc8e91c400f715c" member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } } peers { permanent_uuid: "abb75db3400c4a1784d3cb03cfc744a0" member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } }
20:00:55.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.195003 18139 raft_consensus.cc:374] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:00:55.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.195281 18139 raft_consensus.cc:729] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 716ad9b58f7f4f899cc8e91c400f715c, State: Initialized, Role: FOLLOWER
20:00:55.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.196381 18139 consensus_queue.cc:260] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bef6096de95f42adb4c52fe18c43f55a" member_type: VOTER last_known_addr { host: "127.17.144.62" port: 42077 } } peers { permanent_uuid: "716ad9b58f7f4f899cc8e91c400f715c" member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } } peers { permanent_uuid: "abb75db3400c4a1784d3cb03cfc744a0" member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } }
20:00:55.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.200542 18243 sys_catalog.cc:455] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bef6096de95f42adb4c52fe18c43f55a" member_type: VOTER last_known_addr { host: "127.17.144.62" port: 42077 } } peers { permanent_uuid: "716ad9b58f7f4f899cc8e91c400f715c" member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } } peers { permanent_uuid: "abb75db3400c4a1784d3cb03cfc744a0" member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } } }
20:00:55.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.203299 18139 sys_catalog.cc:564] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c [sys.catalog]: configured and running, proceeding with master startup.
20:00:55.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.203465 18114 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "abb75db3400c4a1784d3cb03cfc744a0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "716ad9b58f7f4f899cc8e91c400f715c" is_pre_election: true
20:00:55.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.204293 18114 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate abb75db3400c4a1784d3cb03cfc744a0 in term 0.
20:00:55.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.206187 18170 leader_election.cc:302] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 716ad9b58f7f4f899cc8e91c400f715c, abb75db3400c4a1784d3cb03cfc744a0; no voters:
20:00:55.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.207353 18228 raft_consensus.cc:2781] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:00:55.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.207796 18228 raft_consensus.cc:482] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:00:55.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.208194 18228 raft_consensus.cc:3037] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [term 0 FOLLOWER]: Advancing to term 1
20:00:55.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.202733 18243 sys_catalog.cc:458] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c [sys.catalog]: This master's current role is: FOLLOWER
20:00:55.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:55.244263 18254 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:00:55.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:55.245031 18254 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:00:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.247718 18228 raft_consensus.cc:504] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bef6096de95f42adb4c52fe18c43f55a" member_type: VOTER last_known_addr { host: "127.17.144.62" port: 42077 } } peers { permanent_uuid: "716ad9b58f7f4f899cc8e91c400f715c" member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } } peers { permanent_uuid: "abb75db3400c4a1784d3cb03cfc744a0" member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } }
20:00:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.249817 18228 leader_election.cc:288] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [CANDIDATE]: Term 1 election: Requested vote from peers bef6096de95f42adb4c52fe18c43f55a (127.17.144.62:42077), 716ad9b58f7f4f899cc8e91c400f715c (127.17.144.61:45409)
20:00:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.250907 18114 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "abb75db3400c4a1784d3cb03cfc744a0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "716ad9b58f7f4f899cc8e91c400f715c"
20:00:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.251489 18114 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c [term 0 FOLLOWER]: Advancing to term 1
20:00:55.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.238830 18066 master_runner.cc:186] Error getting master registration for 127.17.144.62:42077: OK, instance_id { permanent_uuid: "bef6096de95f42adb4c52fe18c43f55a" instance_seqno: 1781899250906606 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:00:55.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.280436 18066 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:00:55.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.329886 18114 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate abb75db3400c4a1784d3cb03cfc744a0 in term 1.
20:00:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.333413 18170 leader_election.cc:302] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 716ad9b58f7f4f899cc8e91c400f715c, abb75db3400c4a1784d3cb03cfc744a0; no voters:
20:00:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.334311 18228 raft_consensus.cc:2781] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [term 1 FOLLOWER]: Leader election won for term 1
20:00:55.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.345295 18228 raft_consensus.cc:686] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [term 1 LEADER]: Becoming Leader. State: Replica: abb75db3400c4a1784d3cb03cfc744a0, State: Running, Role: LEADER
20:00:55.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.346692 18228 consensus_queue.cc:237] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bef6096de95f42adb4c52fe18c43f55a" member_type: VOTER last_known_addr { host: "127.17.144.62" port: 42077 } } peers { permanent_uuid: "716ad9b58f7f4f899cc8e91c400f715c" member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } } peers { permanent_uuid: "abb75db3400c4a1784d3cb03cfc744a0" member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } }
20:00:55.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.353731 18256 sys_catalog.cc:455] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader abb75db3400c4a1784d3cb03cfc744a0. Latest consensus state: current_term: 1 leader_uuid: "abb75db3400c4a1784d3cb03cfc744a0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bef6096de95f42adb4c52fe18c43f55a" member_type: VOTER last_known_addr { host: "127.17.144.62" port: 42077 } } peers { permanent_uuid: "716ad9b58f7f4f899cc8e91c400f715c" member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } } peers { permanent_uuid: "abb75db3400c4a1784d3cb03cfc744a0" member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } } }
20:00:55.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.354549 18256 sys_catalog.cc:458] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [sys.catalog]: This master's current role is: LEADER
20:00:55.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.356931 18258 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
20:00:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.363730 18258 catalog_manager.cc:1484] Initializing Kudu cluster ID...
20:00:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.393829 18114 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c [term 1 FOLLOWER]: Refusing update from remote peer abb75db3400c4a1784d3cb03cfc744a0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:00:55.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.396286 18256 consensus_queue.cc:1035] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "716ad9b58f7f4f899cc8e91c400f715c" member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:00:55.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.425680 18243 sys_catalog.cc:455] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c [sys.catalog]: SysCatalogTable state changed. Reason: New leader abb75db3400c4a1784d3cb03cfc744a0. Latest consensus state: current_term: 1 leader_uuid: "abb75db3400c4a1784d3cb03cfc744a0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bef6096de95f42adb4c52fe18c43f55a" member_type: VOTER last_known_addr { host: "127.17.144.62" port: 42077 } } peers { permanent_uuid: "716ad9b58f7f4f899cc8e91c400f715c" member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } } peers { permanent_uuid: "abb75db3400c4a1784d3cb03cfc744a0" member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } } }
20:00:55.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.426506 18243 sys_catalog.cc:458] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c [sys.catalog]: This master's current role is: FOLLOWER
20:00:55.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.439720 18259 mvcc.cc:205] Tried to move back new op lower bound from 7298659350072102912 to 7298659349910536192. Current Snapshot: MvccSnapshot[applied={T|T < 7298659350072102912}]
20:00:55.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:55.441295 18093 tablet.cc:2369] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:00:55.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.442875 18228 sys_catalog.cc:455] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "abb75db3400c4a1784d3cb03cfc744a0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bef6096de95f42adb4c52fe18c43f55a" member_type: VOTER last_known_addr { host: "127.17.144.62" port: 42077 } } peers { permanent_uuid: "716ad9b58f7f4f899cc8e91c400f715c" member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } } peers { permanent_uuid: "abb75db3400c4a1784d3cb03cfc744a0" member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } } }
20:00:55.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.443686 18228 sys_catalog.cc:458] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [sys.catalog]: This master's current role is: LEADER
20:00:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.446426 18260 mvcc.cc:205] Tried to move back new op lower bound from 7298659350072102912 to 7298659349910536192. Current Snapshot: MvccSnapshot[applied={T|T < 7298659350072102912}]
20:00:55.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.452396 18258 catalog_manager.cc:1347] Generated new cluster ID: ce75345ef9984fd2aeb1dc27529f378c
20:00:55.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.452661 18258 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
20:00:55.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.453817 18243 sys_catalog.cc:455] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "abb75db3400c4a1784d3cb03cfc744a0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bef6096de95f42adb4c52fe18c43f55a" member_type: VOTER last_known_addr { host: "127.17.144.62" port: 42077 } } peers { permanent_uuid: "716ad9b58f7f4f899cc8e91c400f715c" member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } } peers { permanent_uuid: "abb75db3400c4a1784d3cb03cfc744a0" member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } } }
20:00:55.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.454578 18243 sys_catalog.cc:458] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c [sys.catalog]: This master's current role is: FOLLOWER
20:00:55.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.520394 18258 catalog_manager.cc:1370] Generated new certificate authority record
20:00:55.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.523656 18258 catalog_manager.cc:1504] Loading token signing keys...
20:00:55.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.547107 18258 catalog_manager.cc:5868] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0: Generated new TSK 0
20:00:55.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.556586 18258 catalog_manager.cc:1514] Initializing in-progress tserver states...
20:00:55.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.568005 18064 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } attempt: 7
20:00:55.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.603888 17984 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:00:55.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:00:55.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-0/wal
20:00:55.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-0/data
20:00:55.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:00:55.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:00:55.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:00:55.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:55.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:55.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:55.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-0/logs
20:00:55.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-0/data/info.pb
20:00:55.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:55.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:00:55.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:00:55.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:00:55.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:00:55.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:55.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:55.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:55.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:00:55.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:00:55.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.144.1:0
20:00:55.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.144.1
20:00:55.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.144.1
20:00:55.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:55.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.144.62:42077,127.17.144.61:45409,127.17.144.60:42331
20:00:55.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.144.20:44579
20:00:55.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:55.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:55.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:00:55.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.618151 18064 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bef6096de95f42adb4c52fe18c43f55a: Bootstrap starting.
20:00:55.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.625663 18064 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P bef6096de95f42adb4c52fe18c43f55a: Neither blocks nor log segments found. Creating new log.
20:00:55.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.628262 18064 log.cc:824] T 00000000000000000000000000000000 P bef6096de95f42adb4c52fe18c43f55a: Log is configured to *not* fsync() on all Append() calls
20:00:55.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.659547 18064 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bef6096de95f42adb4c52fe18c43f55a: No bootstrap required, opened a new log
20:00:55.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.731763 18064 raft_consensus.cc:348] T 00000000000000000000000000000000 P bef6096de95f42adb4c52fe18c43f55a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bef6096de95f42adb4c52fe18c43f55a" member_type: VOTER last_known_addr { host: "127.17.144.62" port: 42077 } } peers { permanent_uuid: "716ad9b58f7f4f899cc8e91c400f715c" member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } } peers { permanent_uuid: "abb75db3400c4a1784d3cb03cfc744a0" member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } }
20:00:55.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.736135 18064 raft_consensus.cc:374] T 00000000000000000000000000000000 P bef6096de95f42adb4c52fe18c43f55a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:00:55.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.737052 18064 raft_consensus.cc:729] T 00000000000000000000000000000000 P bef6096de95f42adb4c52fe18c43f55a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bef6096de95f42adb4c52fe18c43f55a, State: Initialized, Role: FOLLOWER
20:00:55.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.739068 18064 consensus_queue.cc:260] T 00000000000000000000000000000000 P bef6096de95f42adb4c52fe18c43f55a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bef6096de95f42adb4c52fe18c43f55a" member_type: VOTER last_known_addr { host: "127.17.144.62" port: 42077 } } peers { permanent_uuid: "716ad9b58f7f4f899cc8e91c400f715c" member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } } peers { permanent_uuid: "abb75db3400c4a1784d3cb03cfc744a0" member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } }
20:00:55.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.757932 18267 sys_catalog.cc:455] T 00000000000000000000000000000000 P bef6096de95f42adb4c52fe18c43f55a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bef6096de95f42adb4c52fe18c43f55a" member_type: VOTER last_known_addr { host: "127.17.144.62" port: 42077 } } peers { permanent_uuid: "716ad9b58f7f4f899cc8e91c400f715c" member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } } peers { permanent_uuid: "abb75db3400c4a1784d3cb03cfc744a0" member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } } }
20:00:55.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.759130 18267 sys_catalog.cc:458] T 00000000000000000000000000000000 P bef6096de95f42adb4c52fe18c43f55a [sys.catalog]: This master's current role is: FOLLOWER
20:00:55.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.759734 18064 sys_catalog.cc:564] T 00000000000000000000000000000000 P bef6096de95f42adb4c52fe18c43f55a [sys.catalog]: configured and running, proceeding with master startup.
20:00:55.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.761057 18039 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "abb75db3400c4a1784d3cb03cfc744a0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bef6096de95f42adb4c52fe18c43f55a" is_pre_election: true
20:00:55.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.761991 18039 raft_consensus.cc:2454] T 00000000000000000000000000000000 P bef6096de95f42adb4c52fe18c43f55a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate abb75db3400c4a1784d3cb03cfc744a0 in term 0.
20:00:55.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.763542 18037 raft_consensus.cc:3037] T 00000000000000000000000000000000 P bef6096de95f42adb4c52fe18c43f55a [term 0 FOLLOWER]: Advancing to term 1
20:00:55.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.764199 18038 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "abb75db3400c4a1784d3cb03cfc744a0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "bef6096de95f42adb4c52fe18c43f55a"
20:00:55.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.784834 18037 raft_consensus.cc:1261] T 00000000000000000000000000000000 P bef6096de95f42adb4c52fe18c43f55a [term 1 FOLLOWER]: Refusing update from remote peer abb75db3400c4a1784d3cb03cfc744a0: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:00:55.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.787009 18256 consensus_queue.cc:1035] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "bef6096de95f42adb4c52fe18c43f55a" member_type: VOTER last_known_addr { host: "127.17.144.62" port: 42077 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:00:55.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.867143 18276 sys_catalog.cc:455] T 00000000000000000000000000000000 P bef6096de95f42adb4c52fe18c43f55a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "abb75db3400c4a1784d3cb03cfc744a0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bef6096de95f42adb4c52fe18c43f55a" member_type: VOTER last_known_addr { host: "127.17.144.62" port: 42077 } } peers { permanent_uuid: "716ad9b58f7f4f899cc8e91c400f715c" member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } } peers { permanent_uuid: "abb75db3400c4a1784d3cb03cfc744a0" member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } } }
20:00:55.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.869387 18276 sys_catalog.cc:458] T 00000000000000000000000000000000 P bef6096de95f42adb4c52fe18c43f55a [sys.catalog]: This master's current role is: FOLLOWER
20:00:55.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.868613 18267 sys_catalog.cc:455] T 00000000000000000000000000000000 P bef6096de95f42adb4c52fe18c43f55a [sys.catalog]: SysCatalogTable state changed. Reason: New leader abb75db3400c4a1784d3cb03cfc744a0. Latest consensus state: current_term: 1 leader_uuid: "abb75db3400c4a1784d3cb03cfc744a0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bef6096de95f42adb4c52fe18c43f55a" member_type: VOTER last_known_addr { host: "127.17.144.62" port: 42077 } } peers { permanent_uuid: "716ad9b58f7f4f899cc8e91c400f715c" member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } } peers { permanent_uuid: "abb75db3400c4a1784d3cb03cfc744a0" member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } } }
20:00:55.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.871881 18267 sys_catalog.cc:458] T 00000000000000000000000000000000 P bef6096de95f42adb4c52fe18c43f55a [sys.catalog]: This master's current role is: FOLLOWER
20:00:55.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:55.883941 18284 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:00:56.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:56.251060 18254 catalog_manager.cc:1259] Loaded cluster ID: ce75345ef9984fd2aeb1dc27529f378c
20:00:56.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:56.251871 18254 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c: loading cluster ID for follower catalog manager: success
20:00:56.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:56.259801 18254 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c: acquiring CA information for follower catalog manager: success
20:00:56.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:56.265862 18254 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 716ad9b58f7f4f899cc8e91c400f715c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:00:56.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:56.269634 18265 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:00:56.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:56.270242 18265 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:00:56.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:56.271003 18265 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:00:56.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:56.326916 18265 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:00:56.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:56.328298 18265 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.17.144.1
20:00:56.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:56.341394 18256 sys_catalog.cc:455] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "abb75db3400c4a1784d3cb03cfc744a0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bef6096de95f42adb4c52fe18c43f55a" member_type: VOTER last_known_addr { host: "127.17.144.62" port: 42077 } } peers { permanent_uuid: "716ad9b58f7f4f899cc8e91c400f715c" member_type: VOTER last_known_addr { host: "127.17.144.61" port: 45409 } } peers { permanent_uuid: "abb75db3400c4a1784d3cb03cfc744a0" member_type: VOTER last_known_addr { host: "127.17.144.60" port: 42331 } } }
20:00:56.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:56.342984 18256 sys_catalog.cc:458] T 00000000000000000000000000000000 P abb75db3400c4a1784d3cb03cfc744a0 [sys.catalog]: This master's current role is: LEADER
20:00:56.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:56.389232 18265 tablet_server_runner.cc:78] Tablet server non-default flags:
20:00:56.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.144.20:44579
20:00:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-0/data
20:00:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-0/wal
20:00:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.144.1:0
20:00:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:00:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:56.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:56.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-0/data/info.pb
20:00:56.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.144.1
20:00:56.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:56.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.144.62:42077,127.17.144.61:45409,127.17.144.60:42331
20:00:56.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:00:56.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:56.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:00:56.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:00:56.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:56.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.144.1
20:00:56.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-0/logs
20:00:56.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:56.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:00:56.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:00:56.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:00:56.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:00:56.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:00:56.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:00:56.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 18:51:07 UTC on ccycloud.quasar-lbjuyh.root.comops.site
20:00:56.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456544
20:00:56.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:00:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:56.390791 18265 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:00:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:56.392928 18265 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:00:56.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:56.418900 18292 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:56.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:56.918458 18284 catalog_manager.cc:1259] Loaded cluster ID: ce75345ef9984fd2aeb1dc27529f378c
20:00:56.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:56.928500 18284 catalog_manager.cc:1552] T 00000000000000000000000000000000 P bef6096de95f42adb4c52fe18c43f55a: loading cluster ID for follower catalog manager: success
20:00:56.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:56.955852 18284 catalog_manager.cc:1574] T 00000000000000000000000000000000 P bef6096de95f42adb4c52fe18c43f55a: acquiring CA information for follower catalog manager: success
20:00:56.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:56.974520 18284 catalog_manager.cc:1602] T 00000000000000000000000000000000 P bef6096de95f42adb4c52fe18c43f55a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:00:58.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:57.815392 18291 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 18265
20:00:58.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:56.419963 18293 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:58.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:58.740875 18291 kernel_stack_watchdog.cc:198] Thread 18265 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 395ms:
20:00:58.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:00:58.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:00:58.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:00:58.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:00:58.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:58.741547 18265 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.325s user 0.577s sys 1.077s
20:00:58.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:58.741887 18265 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.325s user 0.577s sys 1.077s
20:00:58.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:58.744071 18295 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:00:58.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:58.746042 18294 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2324 milliseconds
20:00:58.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.746143 18265 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:00:58.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.747951 18265 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:00:58.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.750684 18265 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:00:58.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.752162 18265 hybrid_clock.cc:648] HybridClock initialized: now 1781899258752045 us; error 125 us; skew 500 ppm
20:00:58.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.753316 18265 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:00:58.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.790042 18265 webserver.cc:460] Webserver started at http://127.17.144.1:46575/ using document root <none> and password file <none>
20:00:58.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.792871 18265 fs_manager.cc:359] Metadata directory not provided
20:00:58.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.793157 18265 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:00:58.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.793694 18265 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:00:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.800668 18265 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-0/data/instance:
20:00:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c22aab4e6a154b24919ef6ea2cd29f28"
20:00:58.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:00:58 on dist-test-slave-pdss"
20:00:58.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.802201 18265 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-0/wal/instance:
20:00:58.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c22aab4e6a154b24919ef6ea2cd29f28"
20:00:58.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:00:58 on dist-test-slave-pdss"
20:00:58.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.811964 18265 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.001s sys 0.009s
20:00:58.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.818517 18303 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:00:58.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.819747 18265 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
20:00:58.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.820087 18265 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-0/data,/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-0/wal
20:00:58.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c22aab4e6a154b24919ef6ea2cd29f28"
20:00:58.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:00:58 on dist-test-slave-pdss"
20:00:58.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.820461 18265 fs_report.cc:383] FS layout report
20:00:58.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:00:58.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-0/wal
20:00:58.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-0/wal
20:00:58.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-0/data/data
20:00:58.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:00:58.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:00:58.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:00:58.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:00:58.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:00:58.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:00:58.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:00:58.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:00:58.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:00:58.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:00:58.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:00:58.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.861439 18265 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:00:58.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.862928 18265 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:00:58.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.863423 18265 kserver.cc:163] Server-wide thread pool size limit: 3276
20:00:58.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.866752 18265 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:00:58.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.874040 18265 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:00:58.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.874332 18265 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:00:58.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.874645 18265 ts_tablet_manager.cc:610] Registered 0 tablets
20:00:58.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:58.874845 18265 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:00:59.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:59.173612 18265 rpc_server.cc:307] RPC server started. Bound to: 127.17.144.1:45455
20:00:59.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:59.173722 18416 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.144.1:45455 every 8 connection(s)
20:00:59.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:59.177129 18265 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-0/data/info.pb
20:00:59.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:59.181514 17984 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu as pid 18265
20:00:59.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:59.182325 17984 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-0/wal/instance
20:00:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:59.219851 17984 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:00:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:00:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-1/wal
20:00:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-1/data
20:00:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:00:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:00:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:00:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-1/logs
20:00:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-1/data/info.pb
20:00:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:00:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:00:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:00:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:00:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:00:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:00:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.144.2:0
20:00:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.144.2
20:00:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.144.2
20:00:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.144.62:42077,127.17.144.61:45409,127.17.144.60:42331
20:00:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.144.20:44579
20:00:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:00:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:59.308768 18417 heartbeater.cc:347] Connected to a master server at 127.17.144.60:42331
20:00:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:59.309280 18417 heartbeater.cc:464] Registering TS with master...
20:00:59.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:59.310410 18417 heartbeater.cc:511] Master 127.17.144.60:42331 requested a full tablet report, sending...
20:00:59.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:59.313824 18184 ts_manager.cc:194] Registered new tserver with Master: c22aab4e6a154b24919ef6ea2cd29f28 (127.17.144.1:45455)
20:00:59.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:59.317973 18184 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.17.144.1:47871
20:00:59.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:59.329869 18418 heartbeater.cc:347] Connected to a master server at 127.17.144.61:45409
20:00:59.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:59.330271 18418 heartbeater.cc:464] Registering TS with master...
20:00:59.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:59.330976 18418 heartbeater.cc:511] Master 127.17.144.61:45409 requested a full tablet report, sending...
20:00:59.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:59.332021 18422 heartbeater.cc:347] Connected to a master server at 127.17.144.62:42077
20:00:59.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:59.332338 18422 heartbeater.cc:464] Registering TS with master...
20:00:59.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:59.333005 18422 heartbeater.cc:511] Master 127.17.144.62:42077 requested a full tablet report, sending...
20:00:59.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:59.333880 18104 ts_manager.cc:194] Registered new tserver with Master: c22aab4e6a154b24919ef6ea2cd29f28 (127.17.144.1:45455)
20:00:59.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:59.336076 18029 ts_manager.cc:194] Registered new tserver with Master: c22aab4e6a154b24919ef6ea2cd29f28 (127.17.144.1:45455)
20:00:59.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:59.609395 18421 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:00:59.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:59.609921 18421 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:00:59.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:59.610353 18421 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:00:59.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:59.642987 18421 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:00:59.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:59.643762 18421 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.17.144.2
20:00:59.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:59.678709 18421 tablet_server_runner.cc:78] Tablet server non-default flags:
20:00:59.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:00:59.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.144.20:44579
20:00:59.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:00:59.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:00:59.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-1/data
20:00:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-1/wal
20:00:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:00:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:00:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.144.2:0
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-1/data/info.pb
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.144.2
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.144.62:42077,127.17.144.61:45409,127.17.144.60:42331
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.144.2
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-1/logs
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 18:51:07 UTC on ccycloud.quasar-lbjuyh.root.comops.site
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456544
20:00:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:00:59.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:59.680181 18421 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:00:59.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:00:59.682302 18421 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:00:59.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:59.714972 18434 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:00.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:00.323287 18417 heartbeater.cc:503] Master 127.17.144.60:42331 was elected leader, sending a full tablet report...
20:01:01.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:59.717742 18435 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:01.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:00:59.716162 18437 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:01.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:01.067564 18436 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1351 milliseconds
20:01:01.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.067675 18421 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:01.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.069005 18421 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:01.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.071434 18421 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:01.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.072840 18421 hybrid_clock.cc:648] HybridClock initialized: now 1781899261072767 us; error 59 us; skew 500 ppm
20:01:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.073796 18421 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:01.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.080103 18421 webserver.cc:460] Webserver started at http://127.17.144.2:39553/ using document root <none> and password file <none>
20:01:01.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.081068 18421 fs_manager.cc:359] Metadata directory not provided
20:01:01.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.081303 18421 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:01.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.081790 18421 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:01:01.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.086431 18421 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-1/data/instance:
20:01:01.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "14fc2c361011499ba544a854e7c0887f"
20:01:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:01:01 on dist-test-slave-pdss"
20:01:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.087576 18421 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-1/wal/instance:
20:01:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "14fc2c361011499ba544a854e7c0887f"
20:01:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:01:01 on dist-test-slave-pdss"
20:01:01.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.094967 18421 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.008s
20:01:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.100996 18444 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.102103 18421 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
20:01:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.102383 18421 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-1/data,/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-1/wal
20:01:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "14fc2c361011499ba544a854e7c0887f"
20:01:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:01:01 on dist-test-slave-pdss"
20:01:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.102671 18421 fs_report.cc:383] FS layout report
20:01:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-1/wal
20:01:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-1/wal
20:01:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-1/data/data
20:01:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.160405 18421 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:01:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.162343 18421 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.162753 18421 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:01.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.165215 18421 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:01:01.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.170181 18421 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:01:01.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.170471 18421 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:01:01.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.170801 18421 ts_tablet_manager.cc:610] Registered 0 tablets
20:01:01.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.171032 18421 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:01:01.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.321221 18421 rpc_server.cc:307] RPC server started. Bound to: 127.17.144.2:37383
20:01:01.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.321384 18557 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.144.2:37383 every 8 connection(s)
20:01:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.323984 18421 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-1/data/info.pb
20:01:01.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.325220 17984 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu as pid 18421
20:01:01.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.325789 17984 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-1/wal/instance
20:01:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.350075 17984 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:01:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:01:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-2/wal
20:01:01.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-2/data
20:01:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-2/logs
20:01:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-2/data/info.pb
20:01:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:01:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.144.3:0
20:01:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.144.3
20:01:01.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.144.3
20:01:01.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:01.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.144.62:42077,127.17.144.61:45409,127.17.144.60:42331
20:01:01.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.144.20:44579
20:01:01.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:01.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:01.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:01:01.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.373011 18558 heartbeater.cc:347] Connected to a master server at 127.17.144.60:42331
20:01:01.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.373638 18558 heartbeater.cc:464] Registering TS with master...
20:01:01.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.375139 18558 heartbeater.cc:511] Master 127.17.144.60:42331 requested a full tablet report, sending...
20:01:01.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.378887 18184 ts_manager.cc:194] Registered new tserver with Master: 14fc2c361011499ba544a854e7c0887f (127.17.144.2:37383)
20:01:01.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.381819 18184 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.17.144.2:42017
20:01:01.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.393045 18559 heartbeater.cc:347] Connected to a master server at 127.17.144.61:45409
20:01:01.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.393453 18559 heartbeater.cc:464] Registering TS with master...
20:01:01.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.394605 18559 heartbeater.cc:511] Master 127.17.144.61:45409 requested a full tablet report, sending...
20:01:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.397547 18104 ts_manager.cc:194] Registered new tserver with Master: 14fc2c361011499ba544a854e7c0887f (127.17.144.2:37383)
20:01:01.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.400112 18562 heartbeater.cc:347] Connected to a master server at 127.17.144.62:42077
20:01:01.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.400425 18562 heartbeater.cc:464] Registering TS with master...
20:01:01.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.401057 18562 heartbeater.cc:511] Master 127.17.144.62:42077 requested a full tablet report, sending...
20:01:01.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.403395 18029 ts_manager.cc:194] Registered new tserver with Master: 14fc2c361011499ba544a854e7c0887f (127.17.144.2:37383)
20:01:01.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:01.720986 18565 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:01:01.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:01.721539 18565 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:01.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:01.722031 18565 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:01:01.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:01.757436 18565 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:01:01.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:01.758237 18565 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.17.144.3
20:01:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.793015 18565 tablet_server_runner.cc:78] Tablet server non-default flags:
20:01:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.144.20:44579
20:01:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-2/data
20:01:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-2/wal
20:01:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.144.3:0
20:01:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-2/data/info.pb
20:01:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.144.3
20:01:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.144.62:42077,127.17.144.61:45409,127.17.144.60:42331
20:01:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:01.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:01.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:01.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:01.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.144.3
20:01:01.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-2/logs
20:01:01.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:01.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:01.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:01.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:01:01.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:01:01.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:01:01.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:01.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 18:51:07 UTC on ccycloud.quasar-lbjuyh.root.comops.site
20:01:01.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456544
20:01:01.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:01.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.794381 18565 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:01.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.796159 18565 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:01:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:01.814778 18574 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:02.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:02.387616 18558 heartbeater.cc:503] Master 127.17.144.60:42331 was elected leader, sending a full tablet report...
20:01:03.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:01.815673 18576 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:03.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:01.816587 18573 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:03.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:01.816466 18565 server_base.cc:1034] running on GCE node
20:01:03.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.038868 18565 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.041798 18565 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:03.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.043236 18565 hybrid_clock.cc:648] HybridClock initialized: now 1781899263043200 us; error 51 us; skew 500 ppm
20:01:03.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.044009 18565 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:03.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.051000 18565 webserver.cc:460] Webserver started at http://127.17.144.3:33061/ using document root <none> and password file <none>
20:01:03.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.051892 18565 fs_manager.cc:359] Metadata directory not provided
20:01:03.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.052086 18565 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:03.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.052461 18565 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:01:03.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.057348 18565 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-2/data/instance:
20:01:03.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c54f8c7d0e9f49bc95023c8d8215a0e9"
20:01:03.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:01:03 on dist-test-slave-pdss"
20:01:03.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.058359 18565 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-2/wal/instance:
20:01:03.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c54f8c7d0e9f49bc95023c8d8215a0e9"
20:01:03.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:01:03 on dist-test-slave-pdss"
20:01:03.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.065660 18565 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.002s sys 0.005s
20:01:03.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.072108 18583 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.073140 18565 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
20:01:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.073474 18565 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-2/data,/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-2/wal
20:01:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c54f8c7d0e9f49bc95023c8d8215a0e9"
20:01:03.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:01:03 on dist-test-slave-pdss"
20:01:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.073813 18565 fs_report.cc:383] FS layout report
20:01:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-2/wal
20:01:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-2/wal
20:01:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-2/data/data
20:01:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:03.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:03.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.107923 18565 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:01:03.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.109396 18565 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:03.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.109889 18565 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:03.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.112404 18565 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:01:03.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.117863 18565 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:01:03.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.118099 18565 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:01:03.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.118438 18565 ts_tablet_manager.cc:610] Registered 0 tablets
20:01:03.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.118618 18565 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:01:03.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.281188 18565 rpc_server.cc:307] RPC server started. Bound to: 127.17.144.3:36741
20:01:03.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.281294 18696 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.144.3:36741 every 8 connection(s)
20:01:03.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.283658 18565 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-2/data/info.pb
20:01:03.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.290351 17984 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu as pid 18565
20:01:03.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.290908 17984 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster3617347960384324051/ts-2/wal/instance
20:01:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.355593 18697 heartbeater.cc:347] Connected to a master server at 127.17.144.60:42331
20:01:03.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.356098 18697 heartbeater.cc:464] Registering TS with master...
20:01:03.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.357339 18697 heartbeater.cc:511] Master 127.17.144.60:42331 requested a full tablet report, sending...
20:01:03.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.357883 18698 heartbeater.cc:347] Connected to a master server at 127.17.144.61:45409
20:01:03.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.358614 18698 heartbeater.cc:464] Registering TS with master...
20:01:03.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.359413 18698 heartbeater.cc:511] Master 127.17.144.61:45409 requested a full tablet report, sending...
20:01:03.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.360069 18183 ts_manager.cc:194] Registered new tserver with Master: c54f8c7d0e9f49bc95023c8d8215a0e9 (127.17.144.3:36741)
20:01:03.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.362442 18104 ts_manager.cc:194] Registered new tserver with Master: c54f8c7d0e9f49bc95023c8d8215a0e9 (127.17.144.3:36741)
20:01:03.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.363206 18183 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.17.144.3:58191
20:01:03.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.363412 18699 heartbeater.cc:347] Connected to a master server at 127.17.144.62:42077
20:01:03.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.363734 18699 heartbeater.cc:464] Registering TS with master...
20:01:03.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.364568 18699 heartbeater.cc:511] Master 127.17.144.62:42077 requested a full tablet report, sending...
20:01:03.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.368206 18028 ts_manager.cc:194] Registered new tserver with Master: c54f8c7d0e9f49bc95023c8d8215a0e9 (127.17.144.3:36741)
20:01:03.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:03.370811 17984 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
20:01:03.382 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:01:03.383 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
20:01:03.394 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.17.144.62"
port: 42077
}
bound_http_address {
host: "127.17.144.62"
port: 33523
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.17.144.61"
port: 45409
}
bound_http_address {
host: "127.17.144.61"
port: 38693
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.17.144.60"
port: 42331
}
bound_http_address {
host: "127.17.144.60"
port: 44723
}
}
}
20:01:03.398 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:01:03.401 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.17.144.1"
port: 45455
}
bound_http_address {
host: "127.17.144.1"
port: 46575
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.17.144.2"
port: 37383
}
bound_http_address {
host: "127.17.144.2"
port: 39553
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.17.144.3"
port: 36741
}
bound_http_address {
host: "127.17.144.3"
port: 33061
}
}
}
20:01:03.401 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
20:01:03.590 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
20:01:03.710 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasktTLa0n/test-tmp
20:01:03.711 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:01:03.712 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasktTLa0n/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:01:03.715 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
20:01:04.086 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:04.087 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:01:04.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:04.090126 18708 mini_oidc.cc:152] Starting JWKS server
20:01:04.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:04.094089 18708 webserver.cc:460] Webserver started at http://127.0.0.1:40049/ using document root <none> and password file <none>
20:01:04.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:04.094501 18708 mini_oidc.cc:189] Starting OIDC Discovery server
20:01:04.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:04.095999 18708 webserver.cc:460] Webserver started at http://127.0.0.1:34177/ using document root <none> and password file <none>
20:01:04.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
20:01:04.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/principal' for realm 'KRBTEST.COM',
20:01:04.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
20:01:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:04 dist-test-slave-pdss krb5kdc[18720](info): setting up network...
20:01:04.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
20:01:04.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:04 dist-test-slave-pdss krb5kdc[18720](info): set up 2 sockets
20:01:04.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:04 dist-test-slave-pdss krb5kdc[18720](info): commencing operation
20:01:04.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
20:01:04.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:04.368639 18697 heartbeater.cc:503] Master 127.17.144.60:42331 was elected leader, sending a full tablet report...
20:01:05.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:05.203447 18708 mini_kdc.cc:121] Time spent starting KDC: real 1.082s user 0.011s sys 0.008s
20:01:05.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
20:01:05.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:01:05.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
20:01:05.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
20:01:05.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:01:05.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
20:01:05.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
20:01:05.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:01:05.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
20:01:05.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:01:05.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/test-user.keytab.
20:01:05.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/test-user.keytab.
20:01:05.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:05 dist-test-slave-pdss krb5kdc[18720](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781899265, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:05.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:01:05.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T20:01:05Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
20:01:05.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T20:01:05Z Disabled control of system clock
20:01:05.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.18.69.62@KRBTEST.COM; defaulting to no policy
20:01:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:05.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.18.69.62@KRBTEST.COM" created.
20:01:05.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:05.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.69.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kudu_127.18.69.62.keytab.
20:01:05.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.69.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kudu_127.18.69.62.keytab.
20:01:05.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:05.389998 18708 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:01:05.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:01:05.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/wal
20:01:05.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/data
20:01:05.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:05.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:05.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:05.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:05.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:05.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:05.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/logs
20:01:05.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/data/info.pb
20:01:05.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.69.62:46615
20:01:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.69.62
20:01:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kudu_127.18.69.62.keytab
20:01:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.69.62
20:01:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:05.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:05.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.69.20:44131
20:01:05.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:05.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:05.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:05.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:05.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.69.62:46615,127.18.69.61:38599,127.18.69.60:33643
20:01:05.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:05.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40049/jwks.json
20:01:05.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:05.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:05.701422 18733 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:01:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:05.702194 18733 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:05.702984 18733 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:01:05.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:05.740396 18733 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:05.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:05.740773 18733 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:01:05.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:05.741238 18733 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:01:05.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:05.741616 18733 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:05.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:05.741990 18733 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:01:05.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:05.742394 18733 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:05.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:05.742651 18733 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40049/jwks.json
20:01:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:05.782356 18733 master_runner.cc:386] Master server non-default flags:
20:01:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:05.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.69.20:44131
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/data
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/wal
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.69.62:46615,127.18.69.61:38599,127.18.69.60:33643
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kudu_127.18.69.62.keytab
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.69.62
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.69.62:46615
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40049/jwks.json
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/data/info.pb
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.69.62
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:05.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/logs
20:01:05.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:05.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:05.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:05.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:05.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:01:05.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:01:05.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:05.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 18:51:07 UTC on ccycloud.quasar-lbjuyh.root.comops.site
20:01:05.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456544
20:01:05.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:05.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:05.783855 18733 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:05.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:05.785400 18733 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:01:05.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:05.805467 18740 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:07.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:05.805502 18739 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:07.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:05.806056 18742 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:07.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:07.052872 18741 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:01:07.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.053004 18733 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:07.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.056632 18733 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:07.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.059132 18733 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.060470 18733 hybrid_clock.cc:648] HybridClock initialized: now 1781899267060448 us; error 57 us; skew 500 ppm
20:01:07.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.061393 18733 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:07.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:07 dist-test-slave-pdss krb5kdc[18720](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781899267, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.69.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:07.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.067363 18733 init.cc:377] Logged in from keytab as kudu/127.18.69.62@KRBTEST.COM (short username kudu)
20:01:07.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.072696 18733 webserver.cc:460] Webserver started at http://127.18.69.62:33839/ using document root <none> and password file <none>
20:01:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.073621 18733 fs_manager.cc:359] Metadata directory not provided
20:01:07.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.073796 18733 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:07.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.074159 18733 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:01:07.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.078747 18733 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/data/instance:
20:01:07.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e3f87b808aaf4e9b984cae8492113ec1"
20:01:07.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:01:07 on dist-test-slave-pdss"
20:01:07.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.079754 18733 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/wal/instance:
20:01:07.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e3f87b808aaf4e9b984cae8492113ec1"
20:01:07.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:01:07 on dist-test-slave-pdss"
20:01:07.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.087956 18733 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.000s
20:01:07.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.093971 18752 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:07.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.095042 18733 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
20:01:07.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.095328 18733 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/data,/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/wal
20:01:07.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e3f87b808aaf4e9b984cae8492113ec1"
20:01:07.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:01:07 on dist-test-slave-pdss"
20:01:07.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.095628 18733 fs_report.cc:383] FS layout report
20:01:07.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:07.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/wal
20:01:07.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/wal
20:01:07.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/data/data
20:01:07.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:07.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:07.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:07.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:07.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:07.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:07.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:07.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:07.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:07.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:07.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:07.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.131145 18733 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:01:07.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.137439 18733 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:07.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.137842 18733 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:07.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.217706 18733 rpc_server.cc:307] RPC server started. Bound to: 127.18.69.62:46615
20:01:07.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.217785 18803 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.69.62:46615 every 8 connection(s)
20:01:07.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.220388 18733 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/data/info.pb
20:01:07.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.223845 18708 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu as pid 18733
20:01:07.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.224459 18708 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/wal/instance
20:01:07.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.226516 18804 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:01:07.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.238567 18804 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.69.62" port: 46615 } has no permanent_uuid. Determining permanent_uuid...
20:01:07.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.18.69.61@KRBTEST.COM; defaulting to no policy
20:01:07.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:07.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.18.69.61@KRBTEST.COM" created.
20:01:07.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:07 dist-test-slave-pdss krb5kdc[18720](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781899267, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.69.62@KRBTEST.COM for kudu/127.18.69.62@KRBTEST.COM
20:01:07.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:07.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.69.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kudu_127.18.69.61.keytab.
20:01:07.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.69.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kudu_127.18.69.61.keytab.
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.273557 18708 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/wal
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/data
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/logs
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/data/info.pb
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.69.61:38599
20:01:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.69.61
20:01:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kudu_127.18.69.61.keytab
20:01:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.69.61
20:01:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.69.20:44131
20:01:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.69.62:46615,127.18.69.61:38599,127.18.69.60:33643
20:01:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40049/jwks.json
20:01:07.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.286405 18804 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.69.61" port: 38599 } has no permanent_uuid. Determining permanent_uuid...
20:01:07.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:07.290135 18754 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.69.61:38599: connect: Connection refused (error 111)
20:01:07.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:07.294687 18804 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.69.61:38599: Network error: Client connection negotiation failed: client connection to 127.18.69.61:38599: connect: Connection refused (error 111)
20:01:07.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.344127 18804 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.69.61" port: 38599 } attempt: 1
20:01:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:07.347929 18804 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.69.61:38599: Network error: Client connection negotiation failed: client connection to 127.18.69.61:38599: connect: Connection refused (error 111)
20:01:07.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.416317 18804 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.69.61" port: 38599 } attempt: 2
20:01:07.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:07.420600 18804 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.69.61:38599: Network error: Client connection negotiation failed: client connection to 127.18.69.61:38599: connect: Connection refused (error 111)
20:01:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.512077 18804 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.69.61" port: 38599 } attempt: 3
20:01:07.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:07.515476 18804 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.69.61:38599: Network error: Client connection negotiation failed: client connection to 127.18.69.61:38599: connect: Connection refused (error 111)
20:01:07.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:07.597364 18810 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:01:07.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:07.597893 18810 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:07.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:07.598462 18810 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:01:07.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:07.630664 18810 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:07.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:07.630882 18810 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:01:07.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:07.631210 18810 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:01:07.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:07.631412 18810 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:07.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:07.631608 18810 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:01:07.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:07.631793 18810 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:07.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:07.631930 18810 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40049/jwks.json
20:01:07.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.658941 18804 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.69.61" port: 38599 } attempt: 4
20:01:07.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:07.662870 18804 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.69.61:38599: Network error: Client connection negotiation failed: client connection to 127.18.69.61:38599: connect: Connection refused (error 111)
20:01:07.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.671092 18810 master_runner.cc:386] Master server non-default flags:
20:01:07.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:07.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.69.20:44131
20:01:07.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:07.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:07.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/data
20:01:07.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/wal
20:01:07.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:07.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:07.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.69.62:46615,127.18.69.61:38599,127.18.69.60:33643
20:01:07.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kudu_127.18.69.61.keytab
20:01:07.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.69.61
20:01:07.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:07.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:07.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:07.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.69.61:38599
20:01:07.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:07.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:07.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:07.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40049/jwks.json
20:01:07.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:07.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:07.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:07.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:07.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:07.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/data/info.pb
20:01:07.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.69.61
20:01:07.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/logs
20:01:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:01:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:01:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 18:51:07 UTC on ccycloud.quasar-lbjuyh.root.comops.site
20:01:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456544
20:01:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.672535 18810 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.673915 18810 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:01:07.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:07.693861 18817 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:07.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.962280 18804 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.69.61" port: 38599 } attempt: 5
20:01:07.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:07.965828 18804 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.69.61:38599: Network error: Client connection negotiation failed: client connection to 127.18.69.61:38599: connect: Connection refused (error 111)
20:01:08.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:08.513273 18804 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.69.61" port: 38599 } attempt: 6
20:01:08.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:08.521313 18804 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.69.61:38599: Network error: Client connection negotiation failed: client connection to 127.18.69.61:38599: connect: Connection refused (error 111)
20:01:08.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:07.696296 18810 server_base.cc:1034] running on GCE node
20:01:08.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:07.695371 18820 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:08.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:07.694039 18818 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:08.955482 18810 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:08.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:08.958760 18810 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:08.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:08.960180 18810 hybrid_clock.cc:648] HybridClock initialized: now 1781899268960141 us; error 63 us; skew 500 ppm
20:01:08.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:08.961133 18810 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:08.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:08 dist-test-slave-pdss krb5kdc[18720](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781899268, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.69.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:08.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:08.967562 18810 init.cc:377] Logged in from keytab as kudu/127.18.69.61@KRBTEST.COM (short username kudu)
20:01:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:08.973183 18810 webserver.cc:460] Webserver started at http://127.18.69.61:39933/ using document root <none> and password file <none>
20:01:08.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:08.974280 18810 fs_manager.cc:359] Metadata directory not provided
20:01:08.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:08.974511 18810 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:08.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:08.975009 18810 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:01:08.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:08.979619 18810 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/data/instance:
20:01:08.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df5abcc5dd16419099667e8f47ce0d1e"
20:01:08.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:01:08 on dist-test-slave-pdss"
20:01:08.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:08.980718 18810 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/wal/instance:
20:01:08.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df5abcc5dd16419099667e8f47ce0d1e"
20:01:08.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:01:08 on dist-test-slave-pdss"
20:01:08.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:08.987927 18810 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.000s
20:01:08.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:08.994112 18831 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:08.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:08.995118 18810 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
20:01:08.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:08.995476 18810 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/data,/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/wal
20:01:08.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df5abcc5dd16419099667e8f47ce0d1e"
20:01:08.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:01:08 on dist-test-slave-pdss"
20:01:08.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:08.995821 18810 fs_report.cc:383] FS layout report
20:01:08.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:08.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/wal
20:01:08.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/wal
20:01:08.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/data/data
20:01:08.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:08.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:08.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:08.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:08.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:08.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:08.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:08.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:08.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:08.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:08.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:09.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.029745 18810 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:01:09.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.035694 18810 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:09.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.036214 18810 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:09.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.114886 18810 rpc_server.cc:307] RPC server started. Bound to: 127.18.69.61:38599
20:01:09.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.114959 18883 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.69.61:38599 every 8 connection(s)
20:01:09.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.117763 18810 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/data/info.pb
20:01:09.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.120472 18708 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu as pid 18810
20:01:09.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.121291 18708 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/wal/instance
20:01:09.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.124662 18884 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:01:09.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.18.69.60@KRBTEST.COM; defaulting to no policy
20:01:09.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.137691 18884 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.69.62" port: 46615 } has no permanent_uuid. Determining permanent_uuid...
20:01:09.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:09.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.18.69.60@KRBTEST.COM" created.
20:01:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:01:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.69.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kudu_127.18.69.60.keytab.
20:01:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.69.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kudu_127.18.69.60.keytab.
20:01:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.169770 18708 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:01:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:01:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/wal
20:01:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/data
20:01:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/logs
20:01:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/data/info.pb
20:01:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:09.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:09.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:09.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:09.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:09.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:09.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:09.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:09.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:09.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.69.60:33643
20:01:09.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.69.60
20:01:09.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:09.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kudu_127.18.69.60.keytab
20:01:09.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.69.60
20:01:09.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:09.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:09.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:09.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.69.20:44131
20:01:09.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:09.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:09.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:09.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:09.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.69.62:46615,127.18.69.61:38599,127.18.69.60:33643
20:01:09.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:09.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40049/jwks.json
20:01:09.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:09.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:09 dist-test-slave-pdss krb5kdc[18720](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781899268, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.69.61@KRBTEST.COM for kudu/127.18.69.62@KRBTEST.COM
20:01:09.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.185923 18884 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.69.61" port: 38599 } has no permanent_uuid. Determining permanent_uuid...
20:01:09.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:09 dist-test-slave-pdss krb5kdc[18720](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781899268, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.69.61@KRBTEST.COM for kudu/127.18.69.61@KRBTEST.COM
20:01:09.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.213440 18884 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } has no permanent_uuid. Determining permanent_uuid...
20:01:09.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.216388 18833 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.69.60:33643: connect: Connection refused (error 111)
20:01:09.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.221488 18884 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.69.60:33643: Network error: Client connection negotiation failed: client connection to 127.18.69.60:33643: connect: Connection refused (error 111)
20:01:09.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.271008 18884 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } attempt: 1
20:01:09.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.274722 18884 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.69.60:33643: Network error: Client connection negotiation failed: client connection to 127.18.69.60:33643: connect: Connection refused (error 111)
20:01:09.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.343088 18884 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } attempt: 2
20:01:09.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.347178 18884 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.69.60:33643: Network error: Client connection negotiation failed: client connection to 127.18.69.60:33643: connect: Connection refused (error 111)
20:01:09.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.438547 18884 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } attempt: 3
20:01:09.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.442430 18884 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.69.60:33643: Network error: Client connection negotiation failed: client connection to 127.18.69.60:33643: connect: Connection refused (error 111)
20:01:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.511611 18890 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:01:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.512245 18890 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:09.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.512928 18890 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:01:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.544935 18890 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.545199 18890 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:01:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.545460 18890 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:01:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.545660 18890 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.545856 18890 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:01:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.546074 18890 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.546278 18890 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40049/jwks.json
20:01:09.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.581725 18804 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.69.61" port: 38599 } attempt: 7
20:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.584765 18890 master_runner.cc:386] Master server non-default flags:
20:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.69.20:44131
20:01:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/data
20:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/wal
20:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.69.62:46615,127.18.69.61:38599,127.18.69.60:33643
20:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kudu_127.18.69.60.keytab
20:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.69.60
20:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.69.60:33643
20:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40049/jwks.json
20:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/data/info.pb
20:01:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.69.60
20:01:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:01:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/logs
20:01:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:01:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:01:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 18:51:07 UTC on ccycloud.quasar-lbjuyh.root.comops.site
20:01:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456544
20:01:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.586127 18884 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } attempt: 4
20:01:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.586277 18890 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.587584 18890 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:01:09.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.590651 18884 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.69.60:33643: Network error: Client connection negotiation failed: client connection to 127.18.69.60:33643: connect: Connection refused (error 111)
20:01:09.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:09 dist-test-slave-pdss krb5kdc[18720](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781899267, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.69.62@KRBTEST.COM for kudu/127.18.69.61@KRBTEST.COM
20:01:09.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.617491 18901 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:09.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.619848 18804 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } has no permanent_uuid. Determining permanent_uuid...
20:01:09.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.625120 18804 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.69.60:33643: Network error: Client connection negotiation failed: client connection to 127.18.69.60:33643: connect: Connection refused (error 111)
20:01:09.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.683490 18804 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } attempt: 1
20:01:09.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.687938 18804 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.69.60:33643: Network error: Client connection negotiation failed: client connection to 127.18.69.60:33643: connect: Connection refused (error 111)
20:01:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.769366 18804 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } attempt: 2
20:01:09.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.773012 18804 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.69.60:33643: Network error: Client connection negotiation failed: client connection to 127.18.69.60:33643: connect: Connection refused (error 111)
20:01:09.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.858383 18804 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } attempt: 3
20:01:09.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.862807 18804 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.69.60:33643: Network error: Client connection negotiation failed: client connection to 127.18.69.60:33643: connect: Connection refused (error 111)
20:01:09.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:09.890106 18884 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } attempt: 5
20:01:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.894183 18884 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.69.60:33643: Network error: Client connection negotiation failed: client connection to 127.18.69.60:33643: connect: Connection refused (error 111)
20:01:10.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.003175 18804 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } attempt: 4
20:01:10.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:10.007843 18804 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.69.60:33643: Network error: Client connection negotiation failed: client connection to 127.18.69.60:33643: connect: Connection refused (error 111)
20:01:10.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.291410 18804 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } attempt: 5
20:01:10.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:10.294997 18804 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.69.60:33643: Network error: Client connection negotiation failed: client connection to 127.18.69.60:33643: connect: Connection refused (error 111)
20:01:10.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.441738 18884 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } attempt: 6
20:01:10.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:10.448952 18884 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.69.60:33643: Network error: Client connection negotiation failed: client connection to 127.18.69.60:33643: connect: Connection refused (error 111)
20:01:10.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.847584 18804 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } attempt: 6
20:01:10.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:10.854220 18804 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.69.60:33643: Network error: Client connection negotiation failed: client connection to 127.18.69.60:33643: connect: Connection refused (error 111)
20:01:10.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.617496 18902 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:10.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:09.620654 18904 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:10.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:10.859686 18903 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:01:10.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.859840 18890 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:10.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.863626 18890 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:10.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.866189 18890 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:10.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.867561 18890 hybrid_clock.cc:648] HybridClock initialized: now 1781899270867525 us; error 34 us; skew 500 ppm
20:01:10.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.868391 18890 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:10.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:10 dist-test-slave-pdss krb5kdc[18720](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781899270, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.69.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:10.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.874382 18890 init.cc:377] Logged in from keytab as kudu/127.18.69.60@KRBTEST.COM (short username kudu)
20:01:10.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.879168 18890 webserver.cc:460] Webserver started at http://127.18.69.60:43419/ using document root <none> and password file <none>
20:01:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.880091 18890 fs_manager.cc:359] Metadata directory not provided
20:01:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.880321 18890 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.880858 18890 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:01:10.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.885686 18890 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/data/instance:
20:01:10.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27461d714d2047988f03b80e6fbb75af"
20:01:10.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:01:10 on dist-test-slave-pdss"
20:01:10.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.886818 18890 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/wal/instance:
20:01:10.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27461d714d2047988f03b80e6fbb75af"
20:01:10.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:01:10 on dist-test-slave-pdss"
20:01:10.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.894245 18890 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.003s
20:01:10.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.899864 18920 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.900954 18890 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
20:01:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.901304 18890 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/data,/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/wal
20:01:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27461d714d2047988f03b80e6fbb75af"
20:01:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:01:10 on dist-test-slave-pdss"
20:01:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.901659 18890 fs_report.cc:383] FS layout report
20:01:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/wal
20:01:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/wal
20:01:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/data/data
20:01:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:10.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.948194 18890 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:01:10.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.953930 18890 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:10.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:10.954419 18890 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:11.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.034417 18890 rpc_server.cc:307] RPC server started. Bound to: 127.18.69.60:33643
20:01:11.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.034492 18971 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.69.60:33643 every 8 connection(s)
20:01:11.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.037132 18890 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/data/info.pb
20:01:11.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.042604 18972 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:01:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.046177 18708 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu as pid 18890
20:01:11.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.046669 18708 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/wal/instance
20:01:11.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.060410 18972 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.69.62" port: 46615 } has no permanent_uuid. Determining permanent_uuid...
20:01:11.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:11 dist-test-slave-pdss krb5kdc[18720](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781899270, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.69.60@KRBTEST.COM for kudu/127.18.69.62@KRBTEST.COM
20:01:11.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:11 dist-test-slave-pdss krb5kdc[18720](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781899265, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.18.69.60@KRBTEST.COM
20:01:11.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.098606 18972 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.69.61" port: 38599 } has no permanent_uuid. Determining permanent_uuid...
20:01:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:11 dist-test-slave-pdss krb5kdc[18720](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781899270, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.69.60@KRBTEST.COM for kudu/127.18.69.61@KRBTEST.COM
20:01:11.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:11 dist-test-slave-pdss krb5kdc[18720](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781899265, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.18.69.61@KRBTEST.COM
20:01:11.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.126905 18972 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } has no permanent_uuid. Determining permanent_uuid...
20:01:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:11 dist-test-slave-pdss krb5kdc[18720](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781899270, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.69.60@KRBTEST.COM for kudu/127.18.69.60@KRBTEST.COM
20:01:11.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:11 dist-test-slave-pdss krb5kdc[18720](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781899265, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.18.69.62@KRBTEST.COM
20:01:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.159862 18972 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: Bootstrap starting.
20:01:11.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.168382 18972 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: Neither blocks nor log segments found. Creating new log.
20:01:11.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.168876 18708 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
20:01:11.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.170789 18972 log.cc:824] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: Log is configured to *not* fsync() on all Append() calls
20:01:11.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.176894 18972 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: No bootstrap required, opened a new log
20:01:11.179 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:11.181 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
20:01:11.191 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
20:01:11.193 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
20:01:11.196 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.18.69.62"
port: 46615
}
bound_http_address {
host: "127.18.69.62"
port: 33839
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.18.69.61"
port: 38599
}
bound_http_address {
host: "127.18.69.61"
port: 39933
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.18.69.60"
port: 33643
}
bound_http_address {
host: "127.18.69.60"
port: 43419
}
}
}
20:01:11.196 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:01:11.197 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:11.197 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:01:11.198 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:01:11.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.218006 18972 raft_consensus.cc:348] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3f87b808aaf4e9b984cae8492113ec1" member_type: VOTER last_known_addr { host: "127.18.69.62" port: 46615 } } peers { permanent_uuid: "df5abcc5dd16419099667e8f47ce0d1e" member_type: VOTER last_known_addr { host: "127.18.69.61" port: 38599 } } peers { permanent_uuid: "27461d714d2047988f03b80e6fbb75af" member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } }
20:01:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.218875 18972 raft_consensus.cc:374] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:01:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.219287 18972 raft_consensus.cc:729] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 27461d714d2047988f03b80e6fbb75af, State: Initialized, Role: FOLLOWER
20:01:11.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.220507 18972 consensus_queue.cc:260] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3f87b808aaf4e9b984cae8492113ec1" member_type: VOTER last_known_addr { host: "127.18.69.62" port: 46615 } } peers { permanent_uuid: "df5abcc5dd16419099667e8f47ce0d1e" member_type: VOTER last_known_addr { host: "127.18.69.61" port: 38599 } } peers { permanent_uuid: "27461d714d2047988f03b80e6fbb75af" member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } }
20:01:11.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:11 dist-test-slave-pdss krb5kdc[18720](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781899271, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:11.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:01:11.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.231289 18983 sys_catalog.cc:455] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3f87b808aaf4e9b984cae8492113ec1" member_type: VOTER last_known_addr { host: "127.18.69.62" port: 46615 } } peers { permanent_uuid: "df5abcc5dd16419099667e8f47ce0d1e" member_type: VOTER last_known_addr { host: "127.18.69.61" port: 38599 } } peers { permanent_uuid: "27461d714d2047988f03b80e6fbb75af" member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } } }
20:01:11.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.233160 18983 sys_catalog.cc:458] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [sys.catalog]: This master's current role is: FOLLOWER
20:01:11.233 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:11.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.234225 18972 sys_catalog.cc:564] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [sys.catalog]: configured and running, proceeding with master startup.
20:01:11.235 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/krb5cc
20:01:11.243 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:01:11.245 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/krb5cc
20:01:11.250 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:01:11.254 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
20:01:11.258 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:01:11.259 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:01:11.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:11.286556 18996 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:11.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:11.286844 18996 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:11.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:11 dist-test-slave-pdss krb5kdc[18720](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781899271, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:11.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:01:11.309 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:11.310 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/krb5cc
20:01:11.311 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:01:11.312 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.69.61:38599
20:01:11.314 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
20:01:11.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.315189 18708 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu with pid 18810
20:01:11.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.286434 18890 master_runner.cc:186] Error getting master registration for 127.18.69.62:46615: OK, instance_id { permanent_uuid: "e3f87b808aaf4e9b984cae8492113ec1" instance_seqno: 1781899267195944 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:01:11.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.320731 18890 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:01:11.343 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:11.343 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.69.62:46615
20:01:11.344 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
20:01:11.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.344379 18708 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu with pid 18733
20:01:11.372 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:11.373 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.69.60:33643
20:01:11.373 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
20:01:11.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.373493 18708 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu with pid 18890
20:01:11.394 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:11.394 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.69.61:38599
20:01:11.396 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
20:01:11.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.398239 18708 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:01:11.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/wal
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/data
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/logs
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/data/info.pb
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.69.61:38599
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.69.61
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39933
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kudu_127.18.69.61.keytab
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.69.61
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.69.20:44131
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.69.62:46615,127.18.69.61:38599,127.18.69.60:33643
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:11.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40049/jwks.json
20:01:11.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:11.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:11.723944 18999 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:01:11.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:11.724507 18999 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:11.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:11.725214 18999 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:01:11.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:11.761562 18999 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:11.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:11.761879 18999 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:01:11.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:11.762303 18999 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:01:11.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:11.762530 18999 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:11.762789 18999 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:01:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:11.762987 18999 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:11.763118 18999 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40049/jwks.json
20:01:11.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.800171 18999 master_runner.cc:386] Master server non-default flags:
20:01:11.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:11.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.69.20:44131
20:01:11.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:11.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:11.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/data
20:01:11.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/wal
20:01:11.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:11.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.69.62:46615,127.18.69.61:38599,127.18.69.60:33643
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kudu_127.18.69.61.keytab
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.69.61
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.69.61:38599
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40049/jwks.json
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/data/info.pb
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.69.61
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39933
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/logs
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 18:51:07 UTC on ccycloud.quasar-lbjuyh.root.comops.site
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456544
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.801362 18999 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:11.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.802333 18999 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:01:11.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:11.820829 19007 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:13.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:11.821655 19009 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:13.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:11.822397 18999 server_base.cc:1034] running on GCE node
20:01:13.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:11.822190 19006 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:13.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.029881 18999 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:13.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.032370 18999 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:13.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.033701 18999 hybrid_clock.cc:648] HybridClock initialized: now 1781899273033668 us; error 49 us; skew 500 ppm
20:01:13.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.034475 18999 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:13.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:13 dist-test-slave-pdss krb5kdc[18720](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781899273, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.69.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:13.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.040416 18999 init.cc:377] Logged in from keytab as kudu/127.18.69.61@KRBTEST.COM (short username kudu)
20:01:13.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.045852 18999 webserver.cc:460] Webserver started at http://127.18.69.61:39933/ using document root <none> and password file <none>
20:01:13.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.046756 18999 fs_manager.cc:359] Metadata directory not provided
20:01:13.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.046979 18999 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:13.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.054738 18999 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.001s sys 0.004s
20:01:13.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.058954 19017 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:13.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.059923 18999 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
20:01:13.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.060381 18999 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/data,/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/wal
20:01:13.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df5abcc5dd16419099667e8f47ce0d1e"
20:01:13.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:01:08 on dist-test-slave-pdss"
20:01:13.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.062896 18999 fs_report.cc:383] FS layout report
20:01:13.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:13.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/wal
20:01:13.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/wal
20:01:13.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/data/data
20:01:13.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:13.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:13.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:13.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:13.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:13.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:13.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:13.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:13.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:13.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:13.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:13.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.099638 18999 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:01:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.105950 18999 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.106527 18999 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:13.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.181275 18999 rpc_server.cc:307] RPC server started. Bound to: 127.18.69.61:38599
20:01:13.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.181516 19069 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.69.61:38599 every 8 connection(s)
20:01:13.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.185026 18999 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/data/info.pb
20:01:13.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.190668 18708 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu as pid 18999
20:01:13.191 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:13.191 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.69.62:46615
20:01:13.191 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
20:01:13.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.193329 18708 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:01:13.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:01:13.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/wal
20:01:13.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/data
20:01:13.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:13.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:13.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:13.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:13.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:13.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:13.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/logs
20:01:13.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/data/info.pb
20:01:13.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:13.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:13.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:13.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:13.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:13.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:13.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:13.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:13.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:13.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:13.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:13.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.69.62:46615
20:01:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.69.62
20:01:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33839
20:01:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kudu_127.18.69.62.keytab
20:01:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.69.62
20:01:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.69.20:44131
20:01:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.69.62:46615,127.18.69.61:38599,127.18.69.60:33643
20:01:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40049/jwks.json
20:01:13.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:13.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.199097 19070 sys_catalog.cc:263] Verifying existing consensus state
20:01:13.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260619 20:01:13.199887 19070 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:01:13.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.200330 18999 master.cc:537] Master@127.18.69.61:38599 shutting down...
20:01:13.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.277405 18999 master.cc:552] Master@127.18.69.61:38599 shutdown complete.
20:01:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:01:13.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:13.548735 19072 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:01:13.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:13.549432 19072 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:13.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:13.550184 19072 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:01:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:13.582760 19072 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:13.583037 19072 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:01:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:13.583457 19072 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:01:13.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:13.583822 19072 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:13.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:13.584146 19072 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:01:13.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:13.584468 19072 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:13.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:13.584704 19072 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40049/jwks.json
20:01:13.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.621739 19072 master_runner.cc:386] Master server non-default flags:
20:01:13.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:13.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.69.20:44131
20:01:13.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:13.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:13.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/data
20:01:13.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/wal
20:01:13.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:13.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:13.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.69.62:46615,127.18.69.61:38599,127.18.69.60:33643
20:01:13.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kudu_127.18.69.62.keytab
20:01:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.69.62
20:01:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.69.62:46615
20:01:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40049/jwks.json
20:01:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/data/info.pb
20:01:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.69.62
20:01:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33839
20:01:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/logs
20:01:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:01:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:01:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:13.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 18:51:07 UTC on ccycloud.quasar-lbjuyh.root.comops.site
20:01:13.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456544
20:01:13.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:13.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.623287 19072 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:13.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:13.624593 19072 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:01:13.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:13.641892 19081 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:14.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:13.642124 19078 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:14.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:13.641919 19079 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:14.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:14.891840 19080 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:01:14.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:14.892002 19072 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:01:14.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:14.896255 19072 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:14.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:14.898828 19072 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:14.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:14.900199 19072 hybrid_clock.cc:648] HybridClock initialized: now 1781899274900155 us; error 53 us; skew 500 ppm
20:01:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:14.901051 19072 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:14.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:14 dist-test-slave-pdss krb5kdc[18720](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781899274, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.69.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:14.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:14.910967 19072 init.cc:377] Logged in from keytab as kudu/127.18.69.62@KRBTEST.COM (short username kudu)
20:01:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:14.916718 19072 webserver.cc:460] Webserver started at http://127.18.69.62:33839/ using document root <none> and password file <none>
20:01:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:14.917804 19072 fs_manager.cc:359] Metadata directory not provided
20:01:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:14.918054 19072 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:14.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:14.926458 19072 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.000s sys 0.004s
20:01:14.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:14.931442 19090 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:14.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:14.932646 19072 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
20:01:14.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:14.933030 19072 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/data,/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/wal
20:01:14.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e3f87b808aaf4e9b984cae8492113ec1"
20:01:14.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:01:07 on dist-test-slave-pdss"
20:01:14.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:14.935314 19072 fs_report.cc:383] FS layout report
20:01:14.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:14.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/wal
20:01:14.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/wal
20:01:14.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/data/data
20:01:14.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:14.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:14.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:14.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:14.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:14.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:14.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:14.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:14.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:14.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:14.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:14.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:14.970469 19072 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:01:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:14.975827 19072 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:14.976243 19072 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:15.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:15.049377 19072 rpc_server.cc:307] RPC server started. Bound to: 127.18.69.62:46615
20:01:15.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:15.049449 19141 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.69.62:46615 every 8 connection(s)
20:01:15.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:15.052122 19072 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/data/info.pb
20:01:15.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:15.061239 18708 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu as pid 19072
20:01:15.062 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:15.062 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.69.60:33643
20:01:15.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:15.062822 19142 sys_catalog.cc:263] Verifying existing consensus state
20:01:15.063 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
20:01:15.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260619 20:01:15.063604 19142 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:01:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:15.064035 19072 master.cc:537] Master@127.18.69.62:46615 shutting down...
20:01:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:15.064638 18708 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:01:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu
20:01:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/wal
20:01:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/data
20:01:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:01:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:01:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:01:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/logs
20:01:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/data/info.pb
20:01:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:01:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:01:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:01:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:01:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:01:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:01:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.69.60:33643
20:01:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.69.60
20:01:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43419
20:01:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kudu_127.18.69.60.keytab
20:01:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.69.60
20:01:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:01:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:01:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.69.20:44131
20:01:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.69.62:46615,127.18.69.61:38599,127.18.69.60:33643
20:01:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:15.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40049/jwks.json
20:01:15.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:01:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:15.131500 19072 master.cc:552] Master@127.18.69.62:46615 shutdown complete.
20:01:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:01:15.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:15.409181 19144 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:01:15.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:15.410173 19144 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:01:15.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:15.410887 19144 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:01:15.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:15.446310 19144 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:01:15.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:15.446586 19144 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:01:15.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:15.447073 19144 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:01:15.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:15.447413 19144 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:01:15.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:15.447721 19144 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:01:15.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:15.448035 19144 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:01:15.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:15.448251 19144 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40049/jwks.json
20:01:15.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:15.485764 19144 master_runner.cc:386] Master server non-default flags:
20:01:15.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:01:15.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.69.20:44131
20:01:15.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:01:15.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:01:15.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/data
20:01:15.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/wal
20:01:15.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:01:15.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:01:15.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.69.62:46615,127.18.69.61:38599,127.18.69.60:33643
20:01:15.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/krb5kdc/kudu_127.18.69.60.keytab
20:01:15.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.69.60
20:01:15.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:01:15.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:01:15.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:01:15.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.69.60:33643
20:01:15.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:01:15.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40049/jwks.json
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/data/info.pb
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.69.60
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43419
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/logs
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 18:51:07 UTC on ccycloud.quasar-lbjuyh.root.comops.site
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456544
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:01:15.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:15.487358 19144 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:01:15.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:15.488545 19144 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:01:15.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:15.508296 19151 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:16.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:16.371218 18135 debug-util.cc:398] Leaking SignalData structure 0x7b0800072be0 after lost signal to thread 18070
20:01:16.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:16.371745 18135 debug-util.cc:398] Leaking SignalData structure 0x7b08000af1e0 after lost signal to thread 18138
20:01:16.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:15.508792 19150 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:15.510133 19144 server_base.cc:1034] running on GCE node
20:01:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:15.508306 19153 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:01:16.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:16.744176 19144 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:01:16.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:16.747562 19144 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:01:16.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:16.749197 19144 hybrid_clock.cc:648] HybridClock initialized: now 1781899276749154 us; error 40 us; skew 500 ppm
20:01:16.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:16.750383 19144 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:01:16.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:16 dist-test-slave-pdss krb5kdc[18720](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781899276, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.69.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:16.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:16.757982 19144 init.cc:377] Logged in from keytab as kudu/127.18.69.60@KRBTEST.COM (short username kudu)
20:01:16.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:16.764089 19144 webserver.cc:460] Webserver started at http://127.18.69.60:43419/ using document root <none> and password file <none>
20:01:16.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:16.765379 19144 fs_manager.cc:359] Metadata directory not provided
20:01:16.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:16.765673 19144 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:01:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:16.777254 19144 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.007s sys 0.000s
20:01:16.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:16.783179 19163 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:01:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:16.784583 19144 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
20:01:16.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:16.785025 19144 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/data,/tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/wal
20:01:16.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "27461d714d2047988f03b80e6fbb75af"
20:01:16.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 20:01:10 on dist-test-slave-pdss"
20:01:16.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:16.790122 19144 fs_report.cc:383] FS layout report
20:01:16.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:01:16.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/wal
20:01:16.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/wal
20:01:16.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/data/data
20:01:16.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:01:16.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:01:16.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:01:16.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:01:16.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:01:16.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:01:16.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:01:16.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:01:16.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:01:16.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:01:16.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:01:16.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:16.825935 19144 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:01:16.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:16.831924 19144 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:01:16.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:16.832464 19144 kserver.cc:163] Server-wide thread pool size limit: 3276
20:01:16.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:16.919669 19144 rpc_server.cc:307] RPC server started. Bound to: 127.18.69.60:33643
20:01:16.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:16.919838 19214 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.69.60:33643 every 8 connection(s)
20:01:16.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:16.922930 19144 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasktTLa0n/test-tmp/mini-kudu-cluster2884652732211233914/master-2/data/info.pb
20:01:16.925 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:01:16.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:16.924824 18708 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu as pid 19144
20:01:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:16.937486 19215 sys_catalog.cc:263] Verifying existing consensus state
20:01:16.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:16.946702 19215 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: Bootstrap starting.
20:01:17.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:16.999985 19215 log.cc:824] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: Log is configured to *not* fsync() on all Append() calls
20:01:17.011 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:17.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:17.024585 19215 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:01:17.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:17.025573 19215 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: Bootstrap complete.
20:01:17.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:17.065740 19215 raft_consensus.cc:348] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3f87b808aaf4e9b984cae8492113ec1" member_type: VOTER last_known_addr { host: "127.18.69.62" port: 46615 } } peers { permanent_uuid: "df5abcc5dd16419099667e8f47ce0d1e" member_type: VOTER last_known_addr { host: "127.18.69.61" port: 38599 } } peers { permanent_uuid: "27461d714d2047988f03b80e6fbb75af" member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } }
20:01:17.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:17.069605 19215 raft_consensus.cc:374] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:01:17.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:17.069957 19215 raft_consensus.cc:729] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 27461d714d2047988f03b80e6fbb75af, State: Initialized, Role: FOLLOWER
20:01:17.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:17.071180 19215 consensus_queue.cc:260] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3f87b808aaf4e9b984cae8492113ec1" member_type: VOTER last_known_addr { host: "127.18.69.62" port: 46615 } } peers { permanent_uuid: "df5abcc5dd16419099667e8f47ce0d1e" member_type: VOTER last_known_addr { host: "127.18.69.61" port: 38599 } } peers { permanent_uuid: "27461d714d2047988f03b80e6fbb75af" member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } }
20:01:17.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:17.079728 19219 sys_catalog.cc:455] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3f87b808aaf4e9b984cae8492113ec1" member_type: VOTER last_known_addr { host: "127.18.69.62" port: 46615 } } peers { permanent_uuid: "df5abcc5dd16419099667e8f47ce0d1e" member_type: VOTER last_known_addr { host: "127.18.69.61" port: 38599 } } peers { permanent_uuid: "27461d714d2047988f03b80e6fbb75af" member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } } }
20:01:17.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:17.080770 19219 sys_catalog.cc:458] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [sys.catalog]: This master's current role is: FOLLOWER
20:01:17.081 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:17.083 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.18.69.60:33643(127.18.69.60:33643)] connecting to peer
20:01:17.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:17.083623 19215 sys_catalog.cc:564] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [sys.catalog]: configured and running, proceeding with master startup.
20:01:17.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:17.088642 19169 tablet.cc:2369] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:01:17.137 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.138 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.139 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.139 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.141 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.143 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.69.60:33643(127.18.69.60:33643)] Successfully connected to peer
20:01:17.144 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:17.147433 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:17.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:17.147778 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:17.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:17.139264 19144 master_runner.cc:186] Error getting master registration for 127.18.69.62:46615: Network error: Client connection negotiation failed: client connection to 127.18.69.62:46615: connect: Connection refused (error 111),
20:01:17.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:17.181234 19144 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:01:17.264 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
20:01:17.278 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.18.69.60
20:01:17.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:17.349704 19219 raft_consensus.cc:482] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:01:17.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:17.350286 19219 raft_consensus.cc:504] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3f87b808aaf4e9b984cae8492113ec1" member_type: VOTER last_known_addr { host: "127.18.69.62" port: 46615 } } peers { permanent_uuid: "df5abcc5dd16419099667e8f47ce0d1e" member_type: VOTER last_known_addr { host: "127.18.69.61" port: 38599 } } peers { permanent_uuid: "27461d714d2047988f03b80e6fbb75af" member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } }
20:01:17.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:17.355285 19166 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.69.62:46615: connect: Connection refused (error 111)
20:01:17.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:17.361205 19219 leader_election.cc:288] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e3f87b808aaf4e9b984cae8492113ec1 (127.18.69.62:46615), df5abcc5dd16419099667e8f47ce0d1e (127.18.69.61:38599)
20:01:17.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:17.371730 19165 leader_election.cc:334] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer df5abcc5dd16419099667e8f47ce0d1e (127.18.69.61:38599): Network error: Client connection negotiation failed: client connection to 127.18.69.61:38599: connect: Connection refused (error 111)
20:01:17.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:17.374403 19166 leader_election.cc:334] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e3f87b808aaf4e9b984cae8492113ec1 (127.18.69.62:46615): Network error: Client connection negotiation failed: client connection to 127.18.69.62:46615: connect: Connection refused (error 111)
20:01:17.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:17.379767 19166 leader_election.cc:302] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 27461d714d2047988f03b80e6fbb75af; no voters: df5abcc5dd16419099667e8f47ce0d1e, e3f87b808aaf4e9b984cae8492113ec1
20:01:17.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:17.392130 19219 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:01:17.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:17 dist-test-slave-pdss krb5kdc[18720](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781899271, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.18.69.60@KRBTEST.COM
20:01:17.590 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x03196261, L:/127.0.0.1:38260 - R:/127.18.69.60:33643] using SASL/GSSAPI
20:01:17.611 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:17.616 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:17.618 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:17.622 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.622 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.623 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.626 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.626 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.627 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.629 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:17.637 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:17.638 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:17.639 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.639 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.640 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.647 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.647 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.648 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:17.675 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:17.676 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:17.679 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.679 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.680 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.681 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.681 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.682 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.691 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:17.715 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:17.716 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:17.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.719 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.725 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.726 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.726 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:17.755 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:17.757 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:17.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.759 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.761 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.761 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.762 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.769 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:17.835 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:17.836 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:17.839 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.839 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.840 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.842 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.842 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.843 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.846 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:17.917 [DEBUG - pool-6-thread-1] (NetUtil.java:140) Resolved '127.18.69.62' into [/127.18.69.62] in 1015359ns
20:01:17.919 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:17.924 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.924 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:17.926 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:17.933 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.933 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.933 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:17.939 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:18.095 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:18.096 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:18.098 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:18.099 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:18.100 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:18.101 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:18.102 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:18.103 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:18.104 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:18.152155 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:18.152518 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:18.496 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:18.497 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:18.499 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:18.499 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:18.500 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:18.501 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:18.501 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:18.503 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:18.505 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:18.635 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:18.637 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:18.637 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:18.639 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:18.639 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:18.640 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:18.640 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:18.640 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:18.642 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:19.075 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:19.076 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:19.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:19.076471 19237 raft_consensus.cc:482] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:01:19.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:19.077275 19237 raft_consensus.cc:504] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3f87b808aaf4e9b984cae8492113ec1" member_type: VOTER last_known_addr { host: "127.18.69.62" port: 46615 } } peers { permanent_uuid: "df5abcc5dd16419099667e8f47ce0d1e" member_type: VOTER last_known_addr { host: "127.18.69.61" port: 38599 } } peers { permanent_uuid: "27461d714d2047988f03b80e6fbb75af" member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } }
20:01:19.079 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:19.079 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:19.079 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:19.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:19.079843 19237 leader_election.cc:288] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e3f87b808aaf4e9b984cae8492113ec1 (127.18.69.62:46615), df5abcc5dd16419099667e8f47ce0d1e (127.18.69.61:38599)
20:01:19.086 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:19.086 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:19.087 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:19.087 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:19.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:19.090535 19166 leader_election.cc:334] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e3f87b808aaf4e9b984cae8492113ec1 (127.18.69.62:46615): Network error: Client connection negotiation failed: client connection to 127.18.69.62:46615: connect: Connection refused (error 111)
20:01:19.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:19.091180 19165 leader_election.cc:334] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer df5abcc5dd16419099667e8f47ce0d1e (127.18.69.61:38599): Network error: Client connection negotiation failed: client connection to 127.18.69.61:38599: connect: Connection refused (error 111)
20:01:19.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:19.091549 19165 leader_election.cc:302] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 27461d714d2047988f03b80e6fbb75af; no voters: df5abcc5dd16419099667e8f47ce0d1e, e3f87b808aaf4e9b984cae8492113ec1
20:01:19.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:19.092065 19237 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:01:19.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:19.155699 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:19.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:19.156083 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:20.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:20.158949 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:20.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:20.159215 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:20.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:20.926148 19244 raft_consensus.cc:482] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:01:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:20.926477 19244 raft_consensus.cc:504] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3f87b808aaf4e9b984cae8492113ec1" member_type: VOTER last_known_addr { host: "127.18.69.62" port: 46615 } } peers { permanent_uuid: "df5abcc5dd16419099667e8f47ce0d1e" member_type: VOTER last_known_addr { host: "127.18.69.61" port: 38599 } } peers { permanent_uuid: "27461d714d2047988f03b80e6fbb75af" member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } }
20:01:20.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:20.928215 19244 leader_election.cc:288] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e3f87b808aaf4e9b984cae8492113ec1 (127.18.69.62:46615), df5abcc5dd16419099667e8f47ce0d1e (127.18.69.61:38599)
20:01:20.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:20.934342 19166 leader_election.cc:334] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e3f87b808aaf4e9b984cae8492113ec1 (127.18.69.62:46615): Network error: Client connection negotiation failed: client connection to 127.18.69.62:46615: connect: Connection refused (error 111)
20:01:20.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:20.935021 19165 leader_election.cc:334] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer df5abcc5dd16419099667e8f47ce0d1e (127.18.69.61:38599): Network error: Client connection negotiation failed: client connection to 127.18.69.61:38599: connect: Connection refused (error 111)
20:01:20.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:20.935379 19165 leader_election.cc:302] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 27461d714d2047988f03b80e6fbb75af; no voters: df5abcc5dd16419099667e8f47ce0d1e, e3f87b808aaf4e9b984cae8492113ec1
20:01:20.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:20.935894 19244 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:01:21.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:21.162086 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:21.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:21.162338 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:22.165226 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:22.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:22.165485 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:22.175 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:22.177 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:22.177 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:22.178 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:22.178 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:22.181 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:22.181 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:22.182 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:22.183 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:22.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:22.184131 19144 master_runner.cc:186] Error getting master registration for 127.18.69.62:46615: Network error: Client connection negotiation failed: client connection to 127.18.69.62:46615: connect: Connection refused (error 111),
20:01:22.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:22.184484 19144 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:01:22.835 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:22.836 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:22.838 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:22.838 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:22.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:22.840 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:22.840 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:22.841 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:22.842 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:23.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:23.168308 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:23.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:23.168669 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:23.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:23.789275 19252 raft_consensus.cc:482] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:01:23.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:23.789702 19252 raft_consensus.cc:504] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3f87b808aaf4e9b984cae8492113ec1" member_type: VOTER last_known_addr { host: "127.18.69.62" port: 46615 } } peers { permanent_uuid: "df5abcc5dd16419099667e8f47ce0d1e" member_type: VOTER last_known_addr { host: "127.18.69.61" port: 38599 } } peers { permanent_uuid: "27461d714d2047988f03b80e6fbb75af" member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } }
20:01:23.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:23.791229 19252 leader_election.cc:288] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e3f87b808aaf4e9b984cae8492113ec1 (127.18.69.62:46615), df5abcc5dd16419099667e8f47ce0d1e (127.18.69.61:38599)
20:01:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:23.793378 19166 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.69.62:46615: connect: Connection refused (error 111) [suppressed 5 similar messages]
20:01:23.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:23.795754 19165 leader_election.cc:334] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer df5abcc5dd16419099667e8f47ce0d1e (127.18.69.61:38599): Network error: Client connection negotiation failed: client connection to 127.18.69.61:38599: connect: Connection refused (error 111)
20:01:23.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:23.796144 19166 leader_election.cc:334] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e3f87b808aaf4e9b984cae8492113ec1 (127.18.69.62:46615): Network error: Client connection negotiation failed: client connection to 127.18.69.62:46615: connect: Connection refused (error 111)
20:01:23.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:23.796494 19166 leader_election.cc:302] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 27461d714d2047988f03b80e6fbb75af; no voters: df5abcc5dd16419099667e8f47ce0d1e, e3f87b808aaf4e9b984cae8492113ec1
20:01:23.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:23.797029 19252 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:01:24.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:24.171819 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:24.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:24.172137 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:24.575 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:24.577 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:24.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:24.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:24.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:24.580 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:24.580 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:24.581 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:24.582 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:25.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:25.175081 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:25.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:25.175356 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:25.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:25.682600 18693 debug-util.cc:398] Leaking SignalData structure 0x7b08000be080 after lost signal to thread 18568
20:01:25.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:25.683480 18693 debug-util.cc:398] Leaking SignalData structure 0x7b08000bbd00 after lost signal to thread 18696
20:01:26.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:26.180576 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:26.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:26.181071 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:26.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:26 dist-test-slave-pdss krb5kdc[18720](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781899286, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.69.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:26.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:26.761997 19159 init.cc:295] Successfully reacquired a new kerberos TGT
20:01:26.776 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:26.777 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:26.777 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:26.778 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:26.778 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:26.780 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:26.780 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:26.781 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:26.785 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:27.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:27.185456 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:27.185822 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:27.188133 19144 master_runner.cc:186] Error getting master registration for 127.18.69.62:46615: Network error: Client connection negotiation failed: client connection to 127.18.69.62:46615: connect: Connection refused (error 111),
20:01:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:27.188643 19144 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:01:28.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:28.188619 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:28.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:28.188880 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:29.136 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:29.137 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:29.138 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:29.139 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:29.139 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:29.140 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:29.140 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:29.140 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:29.142 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:29.191468 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:29.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:29.191723 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:29.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:29.830024 19265 raft_consensus.cc:482] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:01:29.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:29.830461 19265 raft_consensus.cc:504] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3f87b808aaf4e9b984cae8492113ec1" member_type: VOTER last_known_addr { host: "127.18.69.62" port: 46615 } } peers { permanent_uuid: "df5abcc5dd16419099667e8f47ce0d1e" member_type: VOTER last_known_addr { host: "127.18.69.61" port: 38599 } } peers { permanent_uuid: "27461d714d2047988f03b80e6fbb75af" member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } }
20:01:29.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:29.831881 19265 leader_election.cc:288] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e3f87b808aaf4e9b984cae8492113ec1 (127.18.69.62:46615), df5abcc5dd16419099667e8f47ce0d1e (127.18.69.61:38599)
20:01:29.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:29.834270 19166 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.69.62:46615: connect: Connection refused (error 111) [suppressed 1 similar messages]
20:01:29.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:29.837669 19166 leader_election.cc:334] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e3f87b808aaf4e9b984cae8492113ec1 (127.18.69.62:46615): Network error: Client connection negotiation failed: client connection to 127.18.69.62:46615: connect: Connection refused (error 111)
20:01:29.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:29.838258 19165 leader_election.cc:334] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer df5abcc5dd16419099667e8f47ce0d1e (127.18.69.61:38599): Network error: Client connection negotiation failed: client connection to 127.18.69.61:38599: connect: Connection refused (error 111)
20:01:29.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:29.838579 19165 leader_election.cc:302] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 27461d714d2047988f03b80e6fbb75af; no voters: df5abcc5dd16419099667e8f47ce0d1e, e3f87b808aaf4e9b984cae8492113ec1
20:01:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:29.839250 19265 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:01:30.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:30.194470 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:30.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:30.194732 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:30.535 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:30.537 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:30.538 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:30.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:30.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:30.540 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:30.540 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:30.540 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:30.543 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:31.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:31.197443 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:31.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:31.197681 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:32.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:32.191318 19144 master_runner.cc:186] Error getting master registration for 127.18.69.62:46615: Network error: Client connection negotiation failed: client connection to 127.18.69.62:46615: connect: Connection refused (error 111),
20:01:32.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:32.191684 19144 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:01:32.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:32.200336 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:32.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:32.200582 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:33.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:33.203342 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:33.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:33.203656 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:34.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:34.206461 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:34.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:34.206707 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:34.355 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:34.356 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:34.357 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:34.357 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:34.357 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:34.359 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:34.359 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:34.359 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:34.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:35.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:35.209532 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:35.209790 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:36.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:36.093300 19276 raft_consensus.cc:482] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:01:36.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:36.093878 19276 raft_consensus.cc:504] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3f87b808aaf4e9b984cae8492113ec1" member_type: VOTER last_known_addr { host: "127.18.69.62" port: 46615 } } peers { permanent_uuid: "df5abcc5dd16419099667e8f47ce0d1e" member_type: VOTER last_known_addr { host: "127.18.69.61" port: 38599 } } peers { permanent_uuid: "27461d714d2047988f03b80e6fbb75af" member_type: VOTER last_known_addr { host: "127.18.69.60" port: 33643 } }
20:01:36.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:36.096757 19276 leader_election.cc:288] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e3f87b808aaf4e9b984cae8492113ec1 (127.18.69.62:46615), df5abcc5dd16419099667e8f47ce0d1e (127.18.69.61:38599)
20:01:36.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:36.099413 19166 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.69.62:46615: connect: Connection refused (error 111) [suppressed 1 similar messages]
20:01:36.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:36.102531 19165 leader_election.cc:334] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer df5abcc5dd16419099667e8f47ce0d1e (127.18.69.61:38599): Network error: Client connection negotiation failed: client connection to 127.18.69.61:38599: connect: Connection refused (error 111)
20:01:36.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:36.103173 19166 leader_election.cc:334] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e3f87b808aaf4e9b984cae8492113ec1 (127.18.69.62:46615): Network error: Client connection negotiation failed: client connection to 127.18.69.62:46615: connect: Connection refused (error 111)
20:01:36.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:36.103490 19166 leader_election.cc:302] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 27461d714d2047988f03b80e6fbb75af; no voters: df5abcc5dd16419099667e8f47ce0d1e, e3f87b808aaf4e9b984cae8492113ec1
20:01:36.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:36.104030 19276 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:01:36.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:36.213109 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:36.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:36.213393 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:36.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 20:01:36 dist-test-slave-pdss krb5kdc[18720](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781899296, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.69.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:01:36.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:36.764366 19159 init.cc:295] Successfully reacquired a new kerberos TGT
20:01:37.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:37.194773 19144 master_runner.cc:186] Error getting master registration for 127.18.69.62:46615: Network error: Client connection negotiation failed: client connection to 127.18.69.62:46615: connect: Connection refused (error 111),
20:01:37.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:37.195263 19144 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:01:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:37.216024 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:37.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:37.216280 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:37.575 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:37.576 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:37.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:37.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:37.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:37.579 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:37.579 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:37.579 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:37.582 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:38.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:38.219143 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:38.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:38.219388 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:39.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:39.222404 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:39.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:39.222702 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:40.225667 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:40.225948 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:40.636 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:40.636 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:40.637 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:40.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:40.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:40.638 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:40.638 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:40.638 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:40.642 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:41.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:41.228575 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:41.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:41.228884 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:42.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:42.197842 19144 master_runner.cc:186] Error getting master registration for 127.18.69.62:46615: Network error: Client connection negotiation failed: client connection to 127.18.69.62:46615: connect: Connection refused (error 111),
20:01:42.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:42.198285 19144 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:01:42.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:42.231825 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:42.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:42.232115 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:42.355 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:42.356 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:42.356 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:42.356 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:42.356 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:42.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:42.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:42.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:42.360 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:43.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:43.234876 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:43.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:43.235131 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:43.256 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:43.256 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:43.257 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:43.257 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:43.257 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:43.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:43.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:43.258 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:43.260 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:44.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:44.237883 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:44.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:44.238147 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:45.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:45.241098 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:45.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:45.241348 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:46.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:46.244731 19232 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 27461d714d2047988f03b80e6fbb75af: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:01:46.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 20:01:46.245129 19232 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:01:46.375 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.61:38599(127.18.69.61:38599)] connecting to peer
20:01:46.375 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.69.62:46615(127.18.69.62:46615)] connecting to peer
20:01:46.376 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:46.376 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.69.61:38599(127.18.69.61:38599)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:46.376 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.61:38599: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599
20:01:46.377 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:46.377 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.62:46615(127.18.69.62:46615)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:46.377 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.69.62:46615: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
20:01:46.380 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643; will retry
20:01:46.384 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29447), Traces: [0ms] refreshing cache from master, [57ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [198ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [197ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [665ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [671ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [674ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [681ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [685ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [688ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [689ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [698ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [706ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [709ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [714ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [734ms] refreshing cache from master, [734ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [738ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [740ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [742ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [748ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [750ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [773ms] refreshing cache from master, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [777ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [783ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [784ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [786ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [814ms] refreshing cache from master, [814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [817ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [820ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [826ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [828ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [893ms] refreshing cache from master, [894ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [898ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [901ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [904ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [905ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [974ms] refreshing cache from master, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [983ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [985ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [986ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [992ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [997ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [999ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1153ms] refreshing cache from master, [1154ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [1155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [1156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [1159ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1161ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [1162ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [1163ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [1554ms] refreshing cache from master, [1555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [1556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [1558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [1558ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [1560ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1563ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [1564ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1694ms] refreshing cache from master, [1694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [1695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [1696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [1698ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [1699ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1701ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [1701ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [2133ms] refreshing cache from master, [2134ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [2135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [2135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [2138ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [2143ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [2145ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [2146ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [5234ms] refreshing cache from master, [5234ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [5235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [5237ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [5237ms] too many traces: truncated at 100 traces, deferred=Deferred@189181498(state=PENDING, result=null, callback=(continuation of Deferred@1248935694 after retry RPC after error@1341992539) -> (continuation of Deferred@1358777221 after retry RPC after error@1853102302) -> (continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019), errback=(continuation of Deferred@1248935694 after retry RPC after error@1341992539) -> (continuation of Deferred@1358777221 after retry RPC after error@1853102302) -> (continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019)))
20:01:46.389 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.18.69.60:33643(127.18.69.60:33643)] disconnecting while in state READY
20:01:46.390 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.18.69.61:38599(127.18.69.61:38599)] disconnecting while in state TERMINATED
20:01:46.390 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.18.69.62:46615(127.18.69.62:46615)] disconnecting while in state TERMINATED
20:01:46.395 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:01:46.397 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.69.60:33643(127.18.69.60:33643)] handling channelInactive
20:01:46.397 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.69.60:33643(127.18.69.60:33643)] cleaning up while in state READY due to: connection closed
20:01:46.397 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:01:46.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:46.399762 18708 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu with pid 19144
20:01:47.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T20:01:47Z chronyd exiting
20:01:48.646 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:01:48.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:48.648083 17984 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu with pid 18265
20:01:48.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:48.682443 17984 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu with pid 18421
20:01:48.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:48.710886 17984 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu with pid 18565
20:01:48.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:48.739517 17984 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu with pid 17996
20:01:48.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:48.770979 17984 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu with pid 18066
20:01:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 20:01:48.801530 17984 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasktTLa0n/build/tsan/bin/kudu with pid 18141
20:01:49.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T20:01:49Z chronyd exiting
20:01:51.005 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29447), Traces: [0ms] refreshing cache from master, [57ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [198ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [197ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [665ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [671ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [674ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [681ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [685ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [688ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [689ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [698ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [706ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [709ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [714ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [734ms] refreshing cache from master, [734ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [738ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [740ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [742ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [748ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [750ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [773ms] refreshing cache from master, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [777ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [783ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [784ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [786ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [814ms] refreshing cache from master, [814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [817ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [820ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [826ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [828ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [893ms] refreshing cache from master, [894ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [898ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [901ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [904ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [905ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [974ms] refreshing cache from master, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [983ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [985ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [986ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [992ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [997ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [999ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1153ms] refreshing cache from master, [1154ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [1155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [1156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [1159ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1161ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [1162ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [1163ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [1554ms] refreshing cache from master, [1555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [1556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [1558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [1558ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [1560ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1563ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [1564ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1694ms] refreshing cache from master, [1694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [1695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [1696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [1698ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [1699ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1701ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [1701ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [2133ms] refreshing cache from master, [2134ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [2135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [2135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [2138ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [2143ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [2145ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [2146ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [5234ms] refreshing cache from master, [5234ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [5235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [5237ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [5237ms] too many traces: truncated at 100 traces, deferred=Deferred@189181498(state=PENDING, result=null, callback=(continuation of Deferred@1248935694 after retry RPC after error@1341992539) -> (continuation of Deferred@1358777221 after retry RPC after error@1853102302) -> (continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019), errback=(continuation of Deferred@1248935694 after retry RPC after error@1341992539) -> (continuation of Deferred@1358777221 after retry RPC after error@1853102302) -> (continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.18.69.62:46615
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29447), Traces: [0ms] refreshing cache from master, [57ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [198ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [197ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [665ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [671ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [674ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [681ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [685ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [688ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [689ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [698ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [706ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [709ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [714ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [734ms] refreshing cache from master, [734ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [738ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [740ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [742ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [748ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [750ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [773ms] refreshing cache from master, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [777ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [783ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [784ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [786ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [814ms] refreshing cache from master, [814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [817ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [820ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [826ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [828ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [893ms] refreshing cache from master, [894ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [898ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [901ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [904ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [905ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [974ms] refreshing cache from master, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [983ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [985ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [986ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [992ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [997ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [999ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1153ms] refreshing cache from master, [1154ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [1155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [1156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [1159ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1161ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [1162ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [1163ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [1554ms] refreshing cache from master, [1555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [1556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [1558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [1558ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [1560ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1563ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [1564ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1694ms] refreshing cache from master, [1694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [1695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [1696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [1698ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [1699ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1701ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [1701ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [2133ms] refreshing cache from master, [2134ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [2135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [2135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [2138ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [2143ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [2145ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [2146ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [5234ms] refreshing cache from master, [5234ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [5235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [5237ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [5237ms] too many traces: truncated at 100 traces, deferred=Deferred@189181498(state=PENDING, result=null, callback=(continuation of Deferred@1248935694 after retry RPC after error@1341992539) -> (continuation of Deferred@1358777221 after retry RPC after error@1853102302) -> (continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019), errback=(continuation of Deferred@1248935694 after retry RPC after error@1341992539) -> (continuation of Deferred@1358777221 after retry RPC after error@1853102302) -> (continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.18.69.62:46615
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
20:01:52.396 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
20:01:52.396 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29447), Traces: [0ms] refreshing cache from master, [57ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [198ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [197ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [665ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [671ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [674ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [681ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [685ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [688ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [689ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [698ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [706ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [709ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [714ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [734ms] refreshing cache from master, [734ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [738ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [740ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [742ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [748ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [750ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [773ms] refreshing cache from master, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [777ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [783ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [784ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [786ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [814ms] refreshing cache from master, [814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [817ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [820ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [826ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [828ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [893ms] refreshing cache from master, [894ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [898ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [901ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [904ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [905ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [974ms] refreshing cache from master, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [983ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [985ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [986ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [992ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [997ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [999ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1153ms] refreshing cache from master, [1154ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [1155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [1156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [1159ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1161ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [1162ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [1163ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [1554ms] refreshing cache from master, [1555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [1556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [1558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [1558ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [1560ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1563ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [1564ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1694ms] refreshing cache from master, [1694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [1695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [1696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [1698ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [1699ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1701ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [1701ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [2133ms] refreshing cache from master, [2134ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [2135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [2135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [2138ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [2143ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [2145ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [2146ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [5234ms] refreshing cache from master, [5234ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [5235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [5237ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [5237ms] too many traces: truncated at 100 traces, deferred=Deferred@189181498(state=PENDING, result=null, callback=(continuation of Deferred@1248935694 after retry RPC after error@1341992539) -> (continuation of Deferred@1358777221 after retry RPC after error@1853102302) -> (continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019), errback=(continuation of Deferred@1248935694 after retry RPC after error@1341992539) -> (continuation of Deferred@1358777221 after retry RPC after error@1853102302) -> (continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.18.69.62:46615
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29447), Traces: [0ms] refreshing cache from master, [57ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [198ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [197ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [665ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [671ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [674ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [681ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [685ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [688ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [689ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [698ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [706ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [709ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [714ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [734ms] refreshing cache from master, [734ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [738ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [740ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [742ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [748ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [750ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [773ms] refreshing cache from master, [774ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [777ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [783ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [784ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [786ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [814ms] refreshing cache from master, [814ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [817ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [820ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [826ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [828ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [893ms] refreshing cache from master, [894ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [898ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [901ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [904ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [905ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [974ms] refreshing cache from master, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [983ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [985ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [986ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [992ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [997ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [999ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1153ms] refreshing cache from master, [1154ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [1155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [1156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [1159ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1161ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [1162ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [1163ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [1554ms] refreshing cache from master, [1555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [1556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [1558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [1558ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [1560ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1563ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [1564ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1694ms] refreshing cache from master, [1694ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [1695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [1696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [1698ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [1699ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [1701ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [1701ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [2133ms] refreshing cache from master, [2134ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [2135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [2135ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.60:33643, [2138ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.62:46615: Network error: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615, [2143ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.60:33643: OK, [2145ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [2146ms] delaying RPC due to: Service unavailable: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [5234ms] refreshing cache from master, [5234ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.61:38599, [5235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.69.62:46615, [5237ms] Sub RPC ConnectToMaster: received response from server master-127.18.69.61:38599: Network error: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599, [5237ms] too many traces: truncated at 100 traces, deferred=Deferred@189181498(state=PENDING, result=null, callback=(continuation of Deferred@1248935694 after retry RPC after error@1341992539) -> (continuation of Deferred@1358777221 after retry RPC after error@1853102302) -> (continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019), errback=(continuation of Deferred@1248935694 after retry RPC after error@1341992539) -> (continuation of Deferred@1358777221 after retry RPC after error@1853102302) -> (continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.18.69.61:38599,127.18.69.62:46615,127.18.69.60:33643) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.61:38599(127.18.69.61:38599): Connection refused: /127.18.69.61:38599,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.69.62:46615(127.18.69.62:46615): Connection refused: /127.18.69.62:46615
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.18.69.62:46615
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)